From 4454b3944098ff79e3a391a97b150337d1ee151a Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Tue, 14 Jul 2026 13:11:09 -0500 Subject: [PATCH 1/5] added warmp up parameter to sh files --- benchmarking/run_custom_dataset.sh | 11 +++++++---- benchmarking/run_real_workload_dataset.sh | 11 +++++++---- benchmarking/run_synthetic_dataset.sh | 3 +++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/benchmarking/run_custom_dataset.sh b/benchmarking/run_custom_dataset.sh index 3810830b0..762e28d2f 100644 --- a/benchmarking/run_custom_dataset.sh +++ b/benchmarking/run_custom_dataset.sh @@ -5,10 +5,11 @@ ulimit -n 4096 python src/evaluator.py \ --mode custom \ --model-name "Meta-Llama-3.3-70B-Instruct" \ ---results-dir "./data/results/llmperf" \ +--results-dir "./data/results" \ --num-concurrent-requests 1 \ --timeout 600 \ ---input-file-path "/benchmarking/prompts/custom_prompt_example.jsonl" \ +--input-file-path "/prompts/custom_prompt_example.jsonl" \ +--num-warmup-requests 0 \ --save-llm-responses False \ --sampling-params '{"max_tokens_to_generate": 256}' \ --use-debugging-mode False \ @@ -24,10 +25,11 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode custom \ # --model-name "Meta-Llama-3.3-70B-Instruct" \ -# --results-dir "./data/results/llmperf" \ +# --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ # --input-file-path "" \ +# --num-warmup-requests 0 \ # --save-llm-responses False \ # --sampling-params '{"max_tokens_to_generate": 256}' \ # --use-debugging-mode False \ @@ -38,10 +40,11 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode custom \ # --model-name "gemma-4-31B-it" \ -# --results-dir "./data/results/llmperf" \ +# --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ # --input-file-path "" \ +# --num-warmup-requests 0 \ # --save-llm-responses False \ # --sampling-params '{"max_tokens_to_generate": 256}' \ # --use-debugging-mode False \ diff --git a/benchmarking/run_real_workload_dataset.sh b/benchmarking/run_real_workload_dataset.sh index 6dfc840d3..62cac5c20 100755 --- a/benchmarking/run_real_workload_dataset.sh +++ b/benchmarking/run_real_workload_dataset.sh @@ -4,8 +4,8 @@ ulimit -n 4096 python src/evaluator.py \ --mode real_workload \ ---model-name "Meta-Llama-3.3-70B-Instruct" \ ---results-dir "./data/results/llmperf" \ +--model-names "Meta-Llama-3.3-70B-Instruct" \ +--results-dir "./data/results" \ --qps 1 \ --qps-distribution "constant" \ --timeout 600 \ @@ -13,6 +13,7 @@ python src/evaluator.py \ --num-output-tokens 1000 \ --multimodal-image-size na \ --num-requests 16 \ +--num-warmup-requests 0 \ --use-debugging-mode False \ --llm-api sncloud @@ -27,13 +28,14 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode real_workload \ # --model-names "Meta-Llama-3.3-70B-Instruct" \ -# --results-dir "./data/results/llmperf" \ +# --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ # --num-input-tokens 1000 \ # --num-output-tokens 1000 \ # --multimodal-image-size na \ # --num-requests 16 \ +# --num-warmup-requests 0 \ # --use-debugging-mode False \ # --llm-api sncloud @@ -42,12 +44,13 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode real_workload \ # --model-names "gemma-4-31B-it" \ -# --results-dir "./data/results/llmperf" \ +# --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ # --num-input-tokens 1000 \ # --num-output-tokens 1000 \ # --multimodal-image-size medium \ # --num-requests 16 \ +# --num-warmup-requests 0 \ # --use-debugging-mode False \ # --llm-api sncloud diff --git a/benchmarking/run_synthetic_dataset.sh b/benchmarking/run_synthetic_dataset.sh index 31e11c8eb..46470f346 100755 --- a/benchmarking/run_synthetic_dataset.sh +++ b/benchmarking/run_synthetic_dataset.sh @@ -12,6 +12,7 @@ python src/evaluator.py \ --num-output-tokens 1000 \ --multimodal-image-size na \ --num-requests 16 \ +--num-warmup-requests 0 \ --use-multiple-prompts False \ --save-llm-responses False \ --use-debugging-mode False \ @@ -36,6 +37,7 @@ python src/evaluator.py \ # --num-output-tokens 1000 \ # --multimodal-image-size na \ # --num-requests 16 \ +# --num-warmup-requests 0 \ # --save-llm-responses False \ # --use-debugging-mode False \ # --llm-api sncloud @@ -53,6 +55,7 @@ python src/evaluator.py \ # --num-output-tokens 1000 \ # --multimodal-image-size medium \ # --num-requests 16 \ +# --num-warmup-requests 0 \ # --save-llm-responses False \ # --use-debugging-mode False \ # --llm-api sncloud From 8dbe95f216daf572ce7cc6982b4bbe6ee5110f51 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Tue, 14 Jul 2026 13:11:31 -0500 Subject: [PATCH 2/5] added warmp up parameter and fixed readme's inconsistencies --- benchmarking/README.md | 33 ++++++++++++++------- benchmarking/benchmarking_bundles/README.md | 27 ++++++++--------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/benchmarking/README.md b/benchmarking/README.md index 319d44314..83f535da6 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -81,7 +81,7 @@ Please follow the instructions [here](../README.md#getting-a-sambanova-api-key-a uv pip install -r requirements.txt ``` -_Note: vLLM is also installed for speed benchmarking. The framework will use your existing SambaNova API credentials. No additional setup is needed beyond the standard environment variables._ +_Note: vLLM is also installed (via `requirements.txt`) for the optional vLLM speed-benchmark comparison. It uses your existing SambaNova API credentials, so no additional setup is needed beyond the standard environment variables._ # Use the starter kit @@ -124,7 +124,7 @@ This option allows you to evaluate the performance of the selected LLM on synthe - **vLLM Only**: Run benchmarks using vLLM's benchmark serve command - **Both (Side-by-Side Comparison)**: Run both benchmarks simultaneously and compare results side-by-side - _Note: vLLM benchmarking requires vLLM to be installed. See the [vLLM Setup](#vllm-setup) section below for installation instructions._ + _Note: vLLM benchmarking requires vLLM, which is listed in `requirements.txt` and installed during [environment setup](#create-the-virtual-environment). For API benchmarking, vLLM drives a remote endpoint, so no local GPU is required._ 2. Enter a model name and choose the right API type @@ -142,7 +142,8 @@ This option allows you to evaluate the performance of the selected LLM on synthe - **Number of output tokens**: The number of output tokens the LLM can generate. *Default*: 1000. - **Number of total requests**: Number of requests sent. *Default*: 10. *Note*: the program can timeout before all requests are sent. Configure the **Timeout** parameter accordingly. - **Number of concurrent requests**: The number of concurrent requests. *Default*: 1. For testing batching-enabled models, this value should be greater than the largest batch_size one needs to test. The typical batch sizes that are supported are 1,4,8 and 16. -- **Timeout**: Number of seconds before program times out. *Default*: 600 seconds +- **Number of warm-up requests**: Number of throwaway requests sent (at the concurrency above) *before* the measured run. Their results are discarded, so one-time costs such as server cold start (weight/KV-cache allocation, autoscaler spin-up) and batch ramp-up don't skew the reported metrics. *Default*: 0 (disabled). *Note*: supported by both the Kit and vLLM benchmarks (the vLLM path maps to vLLM's native `--num-warmups` flag). For the Kit, warm-up shares the same **Timeout** budget as the measured run. +- **Timeout**: Number of seconds before the program times out. *Default*: 600 seconds. *Note*: this is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. 4. Run the performance evaluation @@ -164,7 +165,7 @@ This option allows you to evaluate the performance of the selected LLM on synthe Additionally, if the endpoint supports dynamic batching, the plots will show per-batch metrics. - The results are composed of five plots: + The results are composed of six plots: - ```Distribution of TTFT by batch size```: This bar plot shows the median Time to First Token (TTFT) in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see higher values and higher variance in the client-side metrics compared to the server-side metrics. This difference is mainly due to the request waiting in the queue to be served (for concurrent requests), which is not included in server-side metrics. @@ -172,6 +173,8 @@ This option allows you to evaluate the performance of the selected LLM on synthe - ```Distribution of output throughput by batch size``` plot: This bar plot shows the median number of **output** tokens per second per request in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see good agreement between the client and server-side metrics. For endpoints that support dynamic batching, one should see a decreasing trend in metrics as the batch size increases. + - ```Distribution of Mean Inter-Token Latency (ITL) by batch size```: This bar plot shows the median mean inter-token latency (in milliseconds) per request in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. + - ```Total output throughput per batch size```: This bar plot shows the median total tokens generated per second per batch in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see good agreement between the client and server-side metrics. This metric will calculate the same values as the previous metric for batch size = 1. However, for batch size > 1, it is estimated as the average of ```Output throughput by batch size * Batch size``` for each batch, to account for more generated tokens due to concurrent requests being served in batch mode. - ```LLM requests across time```: This gantt plot shows the duration of the TTFT and end-to-end latency per request in a timeline. One should expect latencies considerably greater than TTFTs, and multiple bars starting at the same time as number of concurrent requests specified. In addition, if the endpoint allows dynamic batching, one could see grouped bars according to the batch sizes supported. @@ -274,7 +277,8 @@ This option allows you to evaluate the performance of the selected LLM on real w - **Number of total requests**: Number of requests sent. *Default*: 10. *Note*: the program can timeout before all requests are sent. Configure the **Timeout** parameter accordingly. - **Queries per second**: the number of queries that will be sent to the endpoint per second. Values QPS<10 are recommended since user can hit rate limits. *Default*: 1.0 - **Queries per second distribution**: the type of wait time distribution in between requests. User can choose the values 'constant', 'uniform', 'exponential'. *Default*: constant. -- **Timeout**: Number of seconds before program times out. *Default*: 600 seconds +- **Number of warm-up requests**: Number of throwaway requests sent *before* the measured run. Their results are discarded, so one-time costs such as server cold start and batch ramp-up don't skew the reported metrics. *Default*: 0 (disabled). *Note*: warm-up requests are fired together (not paced by the QPS above) and share the same **Timeout** budget as the measured run. +- **Timeout**: Number of seconds before the program times out. *Default*: 600 seconds. *Note*: this is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. 4. Run the performance evaluation @@ -336,7 +340,8 @@ This option allows you to evaluate the performance of the selected LLM on your o 4. Set the configuration and tuning parameters - **Number of concurrent requests**: The number of concurrent requests. *Default*: 1. For testing dynamic batching, this value should be greater than the largest batch_size one needs to test. The typical batch sizes that are supported are 1,4,8 and 16. -- **Timeout**: Number of seconds before program times out. *Default*: 600 seconds +- **Num Warm-up Requests**: Number of throwaway requests sent (at the concurrency above) *before* the measured run. Their results are discarded, so one-time costs such as server cold start and batch ramp-up don't skew the reported metrics. *Default*: 0 (disabled). *Note*: warm-up shares the same **Timeout** budget as the measured run. +- **Timeout**: Number of seconds before the program times out. *Default*: 600 seconds. *Note*: this is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. - **Max Output Tokens**: Maximum number of tokens to generate. *Default*: 256 - **Save LLM Responses**: Whether to save the actual outputs of the LLM to an output file. The output file will contain the `response_texts` suffix. @@ -355,6 +360,8 @@ This option allows you to evaluate the performance of the selected LLM on your o The results are composed of five plots: + - ```Distribution of TTFT by batch size```: This bar plot shows the median Time to First Token (TTFT) in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see higher values and higher variance in the client-side metrics compared to the server-side metrics. This difference is mainly due to the request waiting in the queue to be served (for concurrent requests), which is not included in server-side metrics. + - ```Distribution of end-to-end latency by batch size```: This bar plot shows the median end-to-end latency in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see higher values and higher variance in the client-side metrics compared to the server-side metrics. This difference is also mainly due to the request waiting in the queue to be served (for concurrent requests), which is not included in server-side metrics. - ```Distribution of output throughput by batch size``` plot: This bar plot shows the median number of **output** tokens per second per request in a bold colored horizontal line, and a rectangular area representing the range between the 5th and 95th percentile. One should see good agreement between the client and server-side metrics. For endpoints that support dynamic batching, one should see a decreasing trend in metrics as the batch size increases. @@ -416,11 +423,12 @@ You have 3 options for running the program from terminal: _Note: Currently we have specific prompting support for GPT, Llama, Gemma, Mistral, Deepseek, Qwen, Solar, and Eeve. Other instruction models can work, but number of tokens may not be close to the ones specified._ 1. Open the file `run_synthetic_dataset.sh` and configure the following parameters: - - **model-name**: Model name to be used. See section `1. Enter a model name and choose the right API type` in [Synthetic Performance Evaluation](#synthetic-performance-evaluation) for more information about model name. + - **model-names**: Model name(s) to be used. See section `1. Enter a model name and choose the right API type` in [Synthetic Performance Evaluation](#synthetic-performance-evaluation) for more information about model name. - **llm-api**: API type to be chosen. - **results-dir**: Path to the results directory. _Default_: "./data/results/llmperf" - **num-concurrent-requests**: Number of concurrent requests. _Default_: 1. For `vllm` and `both` benchmark modes, this value is used as the request rate passed to `vllm bench serve`. - - **timeout**: Timeout in seconds. _Default_: 600. For `vllm` and `both` benchmark modes, this parameter is not applied on the vLLM runs since it's not supported. + - **num-warmup-requests**: Number of throwaway warm-up requests sent (at the concurrency above) before the measured run. Their results are discarded so server cold-start and batch ramp-up costs don't skew the reported metrics. _Default_: 0 (disabled). _Note_: applies to all benchmark modes — for `vllm`/`both`, it maps to vLLM's native `--num-warmups` flag. For the Kit path it shares the same **timeout** budget as the measured run. + - **timeout**: Timeout in seconds. _Default_: 600. This is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. For `vllm` and `both` benchmark modes, this parameter is not applied on the vLLM runs since it's not supported. - **num-input-tokens**: Number of input tokens to include in the request prompts. It's recommended to choose no more than 2000 tokens to avoid long wait times. _Default_: 1000. - **num-output-tokens**: Number of output tokens in the generation. It is strongly recommended to set this value to no more than 2000, as most LLMs cannot generate outputs beyond this limit. _Default_: 1000. - **multimodal-image-size**: Size of the pre-set image to be used with a **multimodal** model. There are three categories: small (500x500px), medium (1000x1000px) and large (2000x2000px). **Warning!** Multimodal models may activate their guardrails when running benchmarks. Changing the input or output number of tokens may help to solve the issue. If model is not multimodal, then leave the value to na. _Note_: vLLM benchmarking only supports na (text-only). _Default:_ na. @@ -535,12 +543,13 @@ Synthetic prompts for performance evaluation can be found [here](./prompts/). Yo _Note: Currently we have specific prompting support for GPT, Llama, Gemma, Mistral, Deepseek, Qwen, Solar, and Eeve. Other instruction models can work, but number of tokens may not be close to the ones specified._ 1. Open the file `run_real_workload_dataset.sh` and configure the following parameters: - - **model-name**: Model name to be used. See section `1. Enter a model name and choose the right API type` in [Real Workload Evaluation](#real-workload-performance-evaluation) for more information about model name. + - **model-names**: Model name(s) to be used. See section `1. Enter a model name and choose the right API type` in [Real Workload Evaluation](#real-workload-performance-evaluation) for more information about model name. - **llm-api**: API type to be chosen. - **results-dir**: Path to the results directory. _Default_: "./data/results/llmperf" - **qps**: the number of queries that will be sent to the endpoint per second. Values QPS<10 are recommended since user can hit rate limits._Default_: 1 - **qps-distribution**: the type of wait time distribution in between requests. User can choose the values 'constant', 'uniform', 'exponential'. _Default_: constant - - **timeout**: Timeout in seconds. _Default_: 600 + - **num-warmup-requests**: Number of throwaway warm-up requests sent before the measured run. Their results are discarded so server cold-start and batch ramp-up costs don't skew the reported metrics. _Default_: 0 (disabled). _Note_: warm-up requests are fired together (not paced by **qps**) and share the same **timeout** budget as the measured run. + - **timeout**: Timeout in seconds. _Default_: 600. This is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. - **num-input-tokens**: Number of input tokens to include in the request prompts. It's recommended to choose no more than 2000 tokens to avoid long wait times. _Default_: 1000. - **num-output-tokens**: Number of output tokens in the generation. It's recommended to choose no more than 2000 tokens to avoid long wait times. _Default_: 1000. - **multimodal-image-size**: Size of the pre-set image to be used with a **multimodal** model. There are three categories: small (500x500px), medium (1000x1000px) and large (2000x2000px). **Warning!** Multimodal models may activate their guardrails when running benchmarks. Changing the input or output number of tokens may help to solve the issue. If model is not multimodal, then leave the value to na. _Default:_ na. @@ -598,9 +607,11 @@ _Note: Currently we have specific prompting support for GPT, Llama, Gemma, Mistr - **llm-api**: API type to be chosen. - **results-dir**: Path to the results directory. _Default_: "./data/results/llmperf" - **num-concurrent-requests**: Number of concurrent requests. _Default_: 1 - - **timeout**: Timeout in seconds. _Default_: 600 + - **num-warmup-requests**: Number of throwaway warm-up requests sent (at the concurrency above) before the measured run. Their results are discarded so server cold-start and batch ramp-up costs don't skew the reported metrics. _Default_: 0 (disabled). _Note_: warm-up shares the same **timeout** budget as the measured run. + - **timeout**: Timeout in seconds. _Default_: 600. This is a single budget shared across the warm-up and measured phases — if it is reached during warm-up, the measured run is skipped. - **input-file-path**: The location of the custom dataset that you want to evaluate with. You can take as example the file [in here.](./prompts/custom_prompt_example.jsonl) - **save-llm-responses**: Whether to save the actual outputs of the LLM to an output file. The output file will contain the `response_texts` suffix. + - **sampling-params**: Optional JSON string of sampling parameters sent with each request (e.g. `'{"max_tokens_to_generate": 256}'`). _Default_: `{}` (empty). - **use-debugging-mode**: Whether to use the debugging mode or not. WARNING: Debug mode will provide more detailed response at the cost of increased latency. _Default_: False _Note_: You should leave the `--mode` parameter untouched - this indicates what dataset mode to use. diff --git a/benchmarking/benchmarking_bundles/README.md b/benchmarking/benchmarking_bundles/README.md index 79e36e7a9..6e851d791 100644 --- a/benchmarking/benchmarking_bundles/README.md +++ b/benchmarking/benchmarking_bundles/README.md @@ -13,7 +13,7 @@ The required steps for Bundle benchmarking are the following: Modify the following file: -- `/benchmarking/benchmarking_scripts/config.yaml` +- `/benchmarking/benchmarking_bundles/config.yaml` Example: @@ -29,6 +29,8 @@ concurrency_enabled: False max_workers: 4 # Prompt behavior use_multiple_prompts: False +# Batch sizes used to infer batching for the switching-time calculation +batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128] ``` #### Key notes: @@ -37,6 +39,7 @@ use_multiple_prompts: False - max_workers: Maximum number of parallel benchmark jobs. - time_delay: Optional sleep between runs (per row). - use_multiple_prompts: If true, multiple prompts in located in `/benchmarking/prompts/user-prompt_template-text_instruct.yaml` will be used randomly. +- batch_sizes: The allowed batch sizes used to infer batching behavior for the switching-time calculation (see [Batching analysis](#batching-analysis)). Observed request groups are snapped **up** to the nearest value in this list. Defaults to powers of two up to 128 when omitted; add non-power-of-two sizes (e.g. `6`) only when your deployment actually serves those batch sizes. ### 2. Model configuration file @@ -46,7 +49,7 @@ Modify: Header: -`model_name,input_tokens,output_tokens,num_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size` +`model_name,input_tokens,output_tokens,num_requests,num_warmup_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size` Each row defines one benchmark job. @@ -66,39 +69,35 @@ The configuration table in `model_configs_example.csv` details each individual m - `num_requests` Total number of requests sent. - ⚠️ The run may timeout before all requests are sent. - Configure the `timeout` parameter in `benchmarking_bundles/config.yaml` accordingly. +- `num_warmup_requests` + Number of throwaway warm-up requests sent **before** the measured run. Set per + row. To disable, leave the cell blank or `0`. + + Synthetic warm-ups match the test's concurrency level, while real-workload warm-ups ignore the pacing rate and just fire everything immediately. - `concurrent_requests` Enables **synthetic workload benchmarking**. - - - Used when testing batching-enabled models - - Typical values: `1`, `4`, `8`, `16` - If this value is set, `qps` is ignored - `qps` Enables **real workload benchmarking** (queries per second). - - Recommended values `< 10` - Ignored if `concurrent_requests` is set - `qps_distribution` Wait-time distribution between requests. - Supported values: - `constant` (default) - `uniform` - `exponential` - Ignored if concurrent_requests is set + - `multimodal_img_size` Used only for multimodal models to include an image in the benchmark requests. - Supported values: - `small` → 500×500 px - `medium` → 1000×1000 px - `large` → 2000×2000 px - For non-multimodal models, leave it empty. > **Important:** @@ -132,7 +131,7 @@ This mode is useful when: ### 4. Run the benchmark -Once the configuration files are set up, run the benchmark from the **root of the AISK repository**: +Once the configuration files are set up, run the benchmark from the **root of the benchmarking_bundles module**: ```bash bash run_synthetic_perfomance_bundle_eval.sh @@ -202,7 +201,7 @@ Each row in the Excel file corresponds to **one row in `model_configs_example.cs The benchmarking pipeline automatically estimates **batching behavior** by analyzing request timing patterns. - Requests with identical `server_ttft` are grouped together -- Batch size is inferred as the next power of two of the group size +- Batch size is inferred by snapping the group size **up** to the nearest value in the `batch_sizes` list configured in `config.yaml` (defaults to powers of two up to 128) Reported batching fields From 3434da6627c5dacefeddaf25bdc883cf448eb8d7 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Tue, 14 Jul 2026 13:11:45 -0500 Subject: [PATCH 3/5] fixed vllm wording in requirements --- benchmarking/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarking/requirements.txt b/benchmarking/requirements.txt index 797f23382..c273b6d6a 100644 --- a/benchmarking/requirements.txt +++ b/benchmarking/requirements.txt @@ -27,6 +27,7 @@ transformers==4.57.1 joblib==1.4.2 openpyxl==3.1.5 streamlit==1.51.0 -# vLLM (Optional - Linux only, requires NVIDIA GPU with CUDA) -# Uncomment the line below if you're on Linux and want vLLM benchmarking support +# vLLM — used only for the optional vLLM speed-benchmark comparison (benchmark-mode vllm/both). +# For API benchmarking it drives a remote endpoint, so no local GPU is required. +# Native install is best supported on Linux; on other platforms it may require extra steps. vllm>=0.6.0 \ No newline at end of file From 473e6350e91cb2230d0dcaca489d89d5c63281a0 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Tue, 14 Jul 2026 13:14:45 -0500 Subject: [PATCH 4/5] multiple changes to introduce warmp up requests changes --- benchmarking/benchmarking_bundles/config.yaml | 6 +- .../model_configs_example.csv | 14 +- .../synthetic_performance_eval_script.py | 13 +- benchmarking/src/evaluator.py | 14 ++ benchmarking/src/performance_evaluation.py | 159 +++++++++++++++++- benchmarking/src/vllm_benchmark.py | 11 ++ .../pages/custom_performance_eval_st.py | 14 ++ .../streamlit/pages/real_workload_eval_st.py | 15 ++ .../pages/synthetic_performance_eval_st.py | 17 ++ benchmarking/streamlit/streamlit_utils.py | 13 +- 10 files changed, 258 insertions(+), 18 deletions(-) diff --git a/benchmarking/benchmarking_bundles/config.yaml b/benchmarking/benchmarking_bundles/config.yaml index 1f29837b1..54ae72c5d 100644 --- a/benchmarking/benchmarking_bundles/config.yaml +++ b/benchmarking/benchmarking_bundles/config.yaml @@ -1,7 +1,7 @@ -model_configs_path: '/benchmarking/benchmarking_bundles/model_configs_example.csv' +model_configs_path: '/benchmarking_bundles/model_configs_example.csv' llm_api: 'sncloud' -output_files_dir: '/benchmarking/data/bundle_tests/output_files' -consolidated_results_dir: '/benchmarking/data/bundle_tests/consolidated_results' +output_files_dir: '/data/bundle_tests/output_files' +consolidated_results_dir: '/data/bundle_tests/consolidated_results' timeout: 3600 time_delay: 0 # Row-level concurrency diff --git a/benchmarking/benchmarking_bundles/model_configs_example.csv b/benchmarking/benchmarking_bundles/model_configs_example.csv index 90a13300a..807260c01 100644 --- a/benchmarking/benchmarking_bundles/model_configs_example.csv +++ b/benchmarking/benchmarking_bundles/model_configs_example.csv @@ -1,7 +1,7 @@ -model_name,input_tokens,output_tokens,num_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size -Meta-Llama-3.3-70B-Instruct,128,128,1,1,,, -Meta-Llama-3.3-70B-Instruct,128,128,2,2,,, -Meta-Llama-3.3-70B-Instruct,128,128,4,4,,, -Meta-Llama-3.3-70B-Instruct,128,128,8,8,,, -Meta-Llama-3.3-70B-Instruct,128,128,16,16,,, -Meta-Llama-3.3-70B-Instruct,128,128,32,32,,, \ No newline at end of file +model_name,input_tokens,output_tokens,num_requests,num_warmup_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size +Meta-Llama-3.3-70B-Instruct,128,128,1,1,1,,, +Meta-Llama-3.3-70B-Instruct,128,128,2,2,2,,, +Meta-Llama-3.3-70B-Instruct,128,128,4,4,4,,, +Meta-Llama-3.3-70B-Instruct,128,128,8,8,8,,, +Meta-Llama-3.3-70B-Instruct,128,128,16,16,16,,, +Meta-Llama-3.3-70B-Instruct,128,128,32,32,32,,, \ No newline at end of file diff --git a/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py b/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py index e5d37404d..4c5176fe2 100644 --- a/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py +++ b/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py @@ -39,6 +39,7 @@ class ModelConfigRow: input_tokens: int output_tokens: int num_requests: int + num_warmup_requests: int = 0 concurrent_requests: Optional[int] = None qps: Optional[float] = None qps_distribution: str = 'constant' @@ -373,6 +374,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None: model_name = row['model_name'] num_requests = int(row['num_requests']) + num_warmup_requests = int(row.get('num_warmup_requests', 0) or 0) input_tokens = int(row['input_tokens']) output_tokens = int(row['output_tokens']) concurrent_requests = int(row.get('concurrent_requests', 0) or 0) @@ -391,6 +393,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None: user_metadata={'model_idx': 0}, llm_api=self.config['llm_api'], use_multiple_prompts=self.config['use_multiple_prompts'], + num_warmup_requests=num_warmup_requests, ) elif qps: evaluator = RealWorkLoadPerformanceEvaluator( @@ -402,6 +405,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None: timeout=self.config['timeout'], user_metadata={'model_idx': 0}, llm_api=self.config['llm_api'], + num_warmup_requests=num_warmup_requests, ) else: logger.warning(f'Skipping {model_name}: missing concurrency or QPS.') @@ -423,9 +427,12 @@ def run(self, run_name: Optional[str] = None) -> None: from concurrent.futures import ThreadPoolExecutor, as_completed model_configs_df = pd.read_csv(self.config['model_configs_path']) - model_configs_df = model_configs_df.astype( - {'input_tokens': 'Int64', 'output_tokens': 'Int64', 'num_requests': 'Int64'} - ) + int_columns = {'input_tokens': 'Int64', 'output_tokens': 'Int64', 'num_requests': 'Int64'} + # `num_warmup_requests` is an optional per-row column; only cast it when present so older + # CSVs without the column still load. + if 'num_warmup_requests' in model_configs_df.columns: + int_columns['num_warmup_requests'] = 'Int64' + model_configs_df = model_configs_df.astype(int_columns) run_time = datetime.now().strftime('%Y%m%d-%H%M%S.%f') if not run_name: diff --git a/benchmarking/src/evaluator.py b/benchmarking/src/evaluator.py index d564a8b14..ada3c90a3 100644 --- a/benchmarking/src/evaluator.py +++ b/benchmarking/src/evaluator.py @@ -121,6 +121,16 @@ def main() -> None: (default: %(default)s)', ) + parser.add_argument( + '--num-warmup-requests', + type=int, + required=False, + default=0, + help='Number of throwaway warm-up requests to send before the measured run. Warm-up requests are \ + sent at the test concurrency and their results are discarded, absorbing cold-start and batch \ + ramp-up costs so they do not skew the reported metrics. 0 disables warm-up. (default: %(default)s)', + ) + args, _ = parser.parse_known_args() # Parse user metadata. @@ -174,6 +184,7 @@ def main() -> None: save_response_texts=args.save_llm_responses, use_debugging_mode=args.use_debugging_mode, llm_api=args.llm_api, + num_warmup_requests=args.num_warmup_requests, ) # Run performance evaluation @@ -278,6 +289,7 @@ def main() -> None: save_response_texts=args.save_llm_responses, use_debugging_mode=args.use_debugging_mode, llm_api=args.llm_api, + num_warmup_requests=args.num_warmup_requests, ) # Run performance evaluation @@ -300,6 +312,7 @@ def main() -> None: results_dir=args.results_dir, timeout=args.timeout, user_metadata=user_metadata, + num_warmup_requests=args.num_warmup_requests, ) vllm_executor.run_benchmark( @@ -397,6 +410,7 @@ def main() -> None: user_metadata=user_metadata, use_debugging_mode=args.use_debugging_mode, llm_api=args.llm_api, + num_warmup_requests=args.num_warmup_requests, ) # Run performance evaluation diff --git a/benchmarking/src/performance_evaluation.py b/benchmarking/src/performance_evaluation.py index 92b3ed896..ecbbbd45f 100644 --- a/benchmarking/src/performance_evaluation.py +++ b/benchmarking/src/performance_evaluation.py @@ -61,6 +61,7 @@ def __init__( api_variables: Dict[str, str] = {}, is_stream_mode: bool = True, timeout: int = 600, + num_warmup_requests: int = 0, config: Dict[str, Any] = {}, ) -> None: # Set kit's config file @@ -80,10 +81,16 @@ def __init__( self.api_variables = api_variables self.is_stream_mode = is_stream_mode self.timeout = timeout + self.num_warmup_requests = num_warmup_requests + # Absolute monotonic deadline shared across warm-up + measured run. Set at the start of + # each run so a timeout stops everything, in whichever phase it is reached. None => no deadline. + self.deadline: Optional[float] = None self.tokenizer = get_tokenizer(self.model_name) self.stop_event = threading.Event() self.ui_progress_bar = None self.cli_progress_bar = None + # Label shown by the UI progress bar; switched to 'Warming up' during the warm-up phase. + self.progress_phase_label = 'Running requests' self.run_uuid = uuid.uuid4() # To be set upon saving of results @@ -183,11 +190,15 @@ def send_requests( start_time (float): start time of the process num_requests (int): number of total requests """ + # The timeout is a single budget shared across warm-up and the measured run: once the + # shared deadline passes, requests stop regardless of which phase is running. Fall back to + # the legacy per-call bound if no deadline was set. + deadline = self.deadline if self.deadline is not None else (start_time + self.timeout) for request_config in request_config_batch: if self.stop_event.is_set(): logger.info('Stopping request processing in thread due to stop signal.') break - if time.monotonic() - start_time >= self.timeout: + if time.monotonic() >= deadline: break req_metrics, response_text, request_config = llm_request(request_config, self.tokenizer) @@ -204,7 +215,112 @@ def send_requests( if self.cli_progress_bar: self.cli_progress_bar.update(update_unit) if self.ui_progress_bar: - self.ui_progress_bar(len(progress), num_requests) + self.ui_progress_bar(len(progress), num_requests, self.progress_phase_label) + + def build_warmup_configs(self, request_configs: List[RequestConfig]) -> List[RequestConfig]: + """Selects `num_warmup_requests` configs to use for the warm-up phase. + + Warm-up may request more requests than the measured run builds (e.g. warm up 30 requests for a + 10-request test), so the available configs are cycled to reach the requested warm-up count. Each + config is deep-copied before being sent in `run_warmup`, so reusing the same objects is safe. + + Args: + request_configs (List[RequestConfig]): The configs built for the measured run. + + Returns: + List[RequestConfig]: Exactly `num_warmup_requests` configs (empty if warm-up is disabled). + """ + if not self.num_warmup_requests or not request_configs: + return [] + return [request_configs[i % len(request_configs)] for i in range(self.num_warmup_requests)] + + def run_warmup(self, warmup_request_configs: List[RequestConfig]) -> None: + """Sends throwaway requests to warm the server before the measured run. + + Warm-up is executed BEFORE the measurement clock (`start_time`) is captured and its + responses are discarded, so it never enters the metrics summary. Its purpose is to absorb + one-time costs that would otherwise skew the reported latencies/throughput and the inferred + batch size: server-side cold start (weight/KV-cache allocation, graph compilation, autoscaler + spin-up), connection/TLS setup, and the server ramping up to the target batch size. + + Requests are run at the test's concurrency (`num_concurrent_requests`) so the server reaches + the same batching regime as the measured phase. When concurrency is unset (e.g. real workload), + the warm-up requests are simply fanned out together. + + Args: + warmup_request_configs (List[RequestConfig]): Request configs to send as warm-up. + """ + if not warmup_request_configs: + return + + # Deep-copy the configs: sending a request mutates its sampling_params in place + # (e.g. `max_tokens_to_generate` is popped in the client), so warming up on the + # original objects would corrupt the configs the measured run reuses. + warmup_request_configs = [rc.model_copy(deep=True) for rc in warmup_request_configs] + + max_workers = self.num_concurrent_requests or len(warmup_request_configs) + logger.info( + f'Warming up with {len(warmup_request_configs)} request(s) at concurrency {max_workers} ' + '(results discarded)...' + ) + + # Throwaway sinks - these are intentionally never returned or summarized + throwaway_responses: List[LLMResponse] = [] + throwaway_progress: List[Any] = [] + + # Show warm-up progress on its own indicators so the user knows the wait is warm-up, + # not a stall. The CLI gets a dedicated tqdm bar; the Streamlit (UI) callback is kept + # live but re-labelled 'Warming up' and re-scaled to the warm-up request count. All of + # these are restored to their measured-run state afterwards. + saved_cli_progress_bar = self.cli_progress_bar + saved_ui_progress_bar = self.ui_progress_bar + saved_progress_phase_label = self.progress_phase_label + self.cli_progress_bar = tqdm(total=len(warmup_request_configs), desc='Warming Up') + self.progress_phase_label = 'Warming up' + + # Show the warm-up label immediately so the (often slow) first cold-start request doesn't + # leave the UI looking stalled before the first completion updates the bar. + if self.ui_progress_bar: + self.ui_progress_bar(0, len(warmup_request_configs), self.progress_phase_label) + + warmup_start = time.monotonic() + try: + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for request_config in warmup_request_configs: + if self.stop_event.is_set(): + logger.info('Stopping warm-up due to stop signal.') + break + if self.deadline is not None and time.monotonic() >= self.deadline: + logger.warning('Timeout reached during warm-up; stopping warm-up early.') + break + future = executor.submit( + self.send_requests, + [request_config], + throwaway_responses, + throwaway_progress, + warmup_start, + len(warmup_request_configs), + ) + futures.append(future) + for t in executor._threads: + add_script_run_ctx(t) + + for future in as_completed(futures): + try: + future.result() + except Exception as e: + # A failed warm-up request must not abort the benchmark + logger.warning(f'Warm-up request failed (ignored): {e}') + finally: + if self.cli_progress_bar is not None: + self.cli_progress_bar.close() + self.cli_progress_bar = saved_cli_progress_bar + self.ui_progress_bar = saved_ui_progress_bar + self.progress_phase_label = saved_progress_phase_label + + completed = len(throwaway_responses) + logger.info(f'Warm-up complete ({completed}/{len(warmup_request_configs)} sent).') def build_metrics_summary( self, @@ -668,6 +784,19 @@ def get_token_throughput_latencies( llm_responses: List[LLMResponse] = [] progress: List[Any] = [] + # Single timeout budget spanning warm-up + measured run: a timeout stops whichever phase + # is active. Set before warm-up so warm-up time counts against the same budget. + self.deadline = time.monotonic() + self.timeout + + # Warm-up phase (discarded, runs before the measured clock starts) + if self.num_warmup_requests: + self.run_warmup(self.build_warmup_configs(request_configs)) + + # If the shared timeout was exhausted during warm-up, stop before measuring. + if time.monotonic() >= self.deadline: + logger.warning('Timeout reached during warm-up; skipping measured run (no results collected).') + return {}, [] + start_time = time.monotonic() # Use ThreadPoolExecutor to handle threads with ThreadPoolExecutor(max_workers=self.num_concurrent_requests) as executor: @@ -1060,6 +1189,19 @@ def get_token_throughput_latencies( llm_responses: List[LLMResponse] = [] progress: List[Any] = [] + # Single timeout budget spanning warm-up + measured run: a timeout stops whichever phase + # is active. Set before warm-up so warm-up time counts against the same budget. + self.deadline = time.monotonic() + self.timeout + + # Warm-up phase (discarded, runs before the measured clock starts) + if self.num_warmup_requests: + self.run_warmup(self.build_warmup_configs(request_configs)) + + # If the shared timeout was exhausted during warm-up, stop before measuring. + if time.monotonic() >= self.deadline: + logger.warning('Timeout reached during warm-up; skipping measured run (no results collected).') + return {}, [] + start_time = time.monotonic() # Use ThreadPoolExecutor to handle threads with ThreadPoolExecutor(max_workers=self.num_concurrent_requests) as executor: @@ -1392,6 +1534,19 @@ def get_token_throughput_latencies( llm_responses: List[LLMResponse] = [] progress: List[Any] = [] + # Single timeout budget spanning warm-up + measured run: a timeout stops whichever phase + # is active. Set before warm-up so warm-up time counts against the same budget. + self.deadline = time.monotonic() + self.timeout + + # Warm-up phase (discarded, runs before the measured clock starts) + if self.num_warmup_requests: + self.run_warmup(self.build_warmup_configs(request_configs)) + + # If the shared timeout was exhausted during warm-up, stop before measuring. + if time.monotonic() >= self.deadline: + logger.warning('Timeout reached during warm-up; skipping measured run (no results collected).') + return {}, [] + start_time = time.monotonic() # Use ThreadPoolExecutor to handle threads with ThreadPoolExecutor(max_workers=10000) as executor: diff --git a/benchmarking/src/vllm_benchmark.py b/benchmarking/src/vllm_benchmark.py index a6e503413..fb7f8246e 100644 --- a/benchmarking/src/vllm_benchmark.py +++ b/benchmarking/src/vllm_benchmark.py @@ -30,6 +30,7 @@ def __init__( results_dir: str, timeout: int = 600, user_metadata: Optional[Dict[str, Any]] = None, + num_warmup_requests: int = 0, ) -> None: """ Initialize the vLLM benchmark executor. @@ -39,11 +40,15 @@ def __init__( results_dir: Directory to save benchmark results timeout: Timeout for the benchmark in seconds user_metadata: Additional metadata to include in results + num_warmup_requests: Number of throwaway warm-up requests sent before the measured run + via vLLM's native `--num-warmups` flag. vLLM excludes them from the reported metrics + so cold-start and batch ramp-up costs don't skew results. 0 disables warm-up. """ self.model_name = model_name self.results_dir = results_dir self.timeout = timeout self.user_metadata = user_metadata or {} + self.num_warmup_requests = num_warmup_requests self.stop_event = threading.Event() self.result_file_path: Any = None self.individual_responses_file_path = None @@ -189,6 +194,12 @@ def run_benchmark( else: cmd.extend(['--request-rate', '1']) + # Warm-up: vLLM sends these throwaway requests first and excludes them from the reported + # metrics (native `--num-warmups` flag). Only added when enabled, for compatibility with + # older vLLM versions that lack the flag. + if self.num_warmup_requests and self.num_warmup_requests > 0: + cmd.extend(['--num-warmups', str(self.num_warmup_requests)]) + env = os.environ.copy() if api_key: env['OPENAI_API_KEY'] = api_key diff --git a/benchmarking/streamlit/pages/custom_performance_eval_st.py b/benchmarking/streamlit/pages/custom_performance_eval_st.py index 03ffdae5a..773511721 100644 --- a/benchmarking/streamlit/pages/custom_performance_eval_st.py +++ b/benchmarking/streamlit/pages/custom_performance_eval_st.py @@ -102,6 +102,7 @@ def _run_custom_performance_evaluation(progress_bar: Any = None) -> pd.DataFrame model_name=st.session_state.llm, results_dir=results_path, num_concurrent_requests=st.session_state.number_concurrent_requests, + num_warmup_requests=st.session_state.number_warmup_requests, timeout=st.session_state.timeout, input_file_path=st.session_state.file_path, save_response_texts=st.session_state.save_llm_responses, @@ -176,6 +177,19 @@ def main() -> None: disabled=st.session_state.running, ) + st.number_input( + 'Num Warm-up Requests', + min_value=0, + max_value=100, + value=0, + step=1, + key='number_warmup_requests', + disabled=st.session_state.running, + help='Throwaway requests sent (at the concurrency above) before the measured run. Their results ' + 'are discarded so cold-start and batch ramp-up costs do not skew the metrics. 0 disables warm-up. ' + 'Warm-up shares the same timeout budget as the measured run.', + ) + st.number_input( 'Timeout', min_value=60, max_value=1800, value=600, step=1, key='timeout', disabled=st.session_state.running ) diff --git a/benchmarking/streamlit/pages/real_workload_eval_st.py b/benchmarking/streamlit/pages/real_workload_eval_st.py index 2448ebc36..c3cafefe1 100644 --- a/benchmarking/streamlit/pages/real_workload_eval_st.py +++ b/benchmarking/streamlit/pages/real_workload_eval_st.py @@ -59,6 +59,8 @@ def _initialize_session_variables() -> None: st.session_state.number_requests = None if 'number_concurrent_requests' not in st.session_state: st.session_state.number_concurrent_requests = None + if 'number_warmup_requests' not in st.session_state: + st.session_state.number_warmup_requests = None if 'timeout' not in st.session_state: st.session_state.timeout = None if 'llm_api' not in st.session_state: @@ -105,6 +107,7 @@ def _run_performance_evaluation(progress_bar: Any = None) -> pd.DataFrame: multimodal_image_size=st.session_state.multimodal_image_size, qps=st.session_state.qps, qps_distribution=st.session_state.qps_distribution, + num_warmup_requests=st.session_state.number_warmup_requests, timeout=st.session_state.timeout, llm_api=st.session_state.llm_api, api_variables=api_variables, @@ -227,6 +230,18 @@ def main() -> None: disabled=st.session_state.running, ) + st.session_state.number_warmup_requests = st.number_input( + 'Number of warm-up requests', + min_value=0, + max_value=100, + value=0, + step=1, + disabled=st.session_state.running, + help='Throwaway requests sent before the measured run, discarded so cold-start and batch ramp-up ' + 'costs do not skew the metrics. 0 disables warm-up. Warm-up shares the same timeout budget as the ' + 'measured run. Note: warm-up requests are fired together (not paced by the QPS above).', + ) + st.session_state.timeout = st.number_input( 'Timeout', min_value=60, max_value=1800, value=600, step=1, disabled=st.session_state.running ) diff --git a/benchmarking/streamlit/pages/synthetic_performance_eval_st.py b/benchmarking/streamlit/pages/synthetic_performance_eval_st.py index 099123bbb..802b15686 100644 --- a/benchmarking/streamlit/pages/synthetic_performance_eval_st.py +++ b/benchmarking/streamlit/pages/synthetic_performance_eval_st.py @@ -70,6 +70,8 @@ def _initialize_session_variables() -> None: st.session_state.number_requests = None if 'number_concurrent_requests' not in st.session_state: st.session_state.number_concurrent_requests = None + if 'number_warmup_requests' not in st.session_state: + st.session_state.number_warmup_requests = None if 'timeout' not in st.session_state: st.session_state.timeout = None if 'llm_api' not in st.session_state: @@ -171,6 +173,7 @@ def _run_performance_evaluation(progress_bar: Any = None) -> pd.DataFrame: results_dir=results_path, multimodal_image_size=st.session_state.multimodal_image_size, num_concurrent_requests=st.session_state.number_concurrent_requests, + num_warmup_requests=st.session_state.number_warmup_requests, timeout=st.session_state.timeout, llm_api=st.session_state.llm_api, api_variables=api_variables, @@ -217,6 +220,7 @@ def _run_vllm_benchmark(progress_bar: Any = None) -> pd.DataFrame: results_dir=results_path, timeout=st.session_state.timeout, user_metadata={'model_idx': 0}, + num_warmup_requests=st.session_state.number_warmup_requests, ) st.session_state.vllm_evaluator.run_benchmark( @@ -463,6 +467,19 @@ def main() -> None: disabled=st.session_state.running or st.session_state.optional_download, ) + st.session_state.number_warmup_requests = st.number_input( + 'Number of warm-up requests', + min_value=0, + max_value=100, + value=0, + step=1, + disabled=st.session_state.running or st.session_state.optional_download, + help='Throwaway requests sent (at the concurrency above) before the measured run. Their results ' + 'are discarded so cold-start and batch ramp-up costs do not skew the metrics. 0 disables warm-up. ' + 'Supported by both the Kit and vLLM benchmarks; for the Kit it shares the same timeout budget as ' + 'the measured run.', + ) + timeout_blocked = st.session_state.benchmark_mode in ['vllm', 'both'] st.session_state.timeout = st.number_input( 'Timeout', diff --git a/benchmarking/streamlit/streamlit_utils.py b/benchmarking/streamlit/streamlit_utils.py index 0b86a418e..c446c3f5e 100644 --- a/benchmarking/streamlit/streamlit_utils.py +++ b/benchmarking/streamlit/streamlit_utils.py @@ -208,9 +208,16 @@ def find_pages_to_show() -> List[Any]: return pages_to_show -def update_progress_bar(step: int, total_steps: int) -> None: - """Update the progress bar.""" - st.session_state.progress_bar.progress(value=step / total_steps, text=f'Running requests: {step}/{total_steps}') +def update_progress_bar(step: int, total_steps: int, phase: str = 'Running requests') -> None: + """Update the progress bar. + + Args: + step: Number of completed steps. + total_steps: Total number of steps for the current phase. + phase: Label for the current phase (e.g. 'Warming up' or 'Running requests'). + """ + fraction = step / total_steps if total_steps else 0 + st.session_state.progress_bar.progress(value=fraction, text=f'{phase}: {step}/{total_steps}') def set_api_variables() -> Dict[str, Any]: From f7cc324a1c9ae57608c56f1b9cd9009ca9482ee4 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Tue, 14 Jul 2026 16:23:54 -0500 Subject: [PATCH 5/5] patched the utils test for last chunk when choices is empty --- utils/tests/api_testing.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/tests/api_testing.py b/utils/tests/api_testing.py index be261304c..6f9b709d9 100644 --- a/utils/tests/api_testing.py +++ b/utils/tests/api_testing.py @@ -113,11 +113,15 @@ def test_client_chat_completion_text_streaming(self) -> None: messages=[{'role': 'user', 'content': text_prompt}], stream=True, ) - + for chunk in response: self.assertTrue(hasattr(chunk, 'id')) self.assertTrue(hasattr(chunk, 'choices')) - self.assertIsInstance(chunk.choices[0].delta.content, str) + if chunk.choices: + self.assertIsInstance(chunk.choices[0].delta.content, str) + else: + # Final usage chunk + self.assertIsNotNone(chunk.usage) self.assertTrue(hasattr(chunk, 'model')) self.assertIn(chunk.model, [model, f'{model}-Text']) self.assertTrue(hasattr(chunk, 'usage'))