Feature: add warmp up requests in benchmarking kit

benchmarking/benchmarking_bundles/README.md

@@ -13,7 +13,7 @@ The required steps for Bundle benchmarking are the following:
 
 Modify the following file:
 
-- `<PATH TO AISK REPO HERE>/benchmarking/benchmarking_scripts/config.yaml`
+- `<PATH TO AISK REPO HERE>/benchmarking/benchmarking_bundles/config.yaml`
 
 Example:
 
@@ -29,6 +29,8 @@ concurrency_enabled: False
 max_workers: 4
 # Prompt behavior
 use_multiple_prompts: False
+# Batch sizes used to infer batching for the switching-time calculation
+batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128]
 ```
 
 #### Key notes:
@@ -37,6 +39,7 @@ use_multiple_prompts: False
 - max_workers: Maximum number of parallel benchmark jobs.
 - time_delay: Optional sleep between runs (per row).
 - use_multiple_prompts: If true, multiple prompts in located in `<PATH TO AISK REPO>/benchmarking/prompts/user-prompt_template-text_instruct.yaml` will be used randomly.
+- batch_sizes: The allowed batch sizes used to infer batching behavior for the switching-time calculation (see [Batching analysis](#batching-analysis)). Observed request groups are snapped **up** to the nearest value in this list. Defaults to powers of two up to 128 when omitted; add non-power-of-two sizes (e.g. `6`) only when your deployment actually serves those batch sizes.
 
 ### 2. Model configuration file
 
@@ -46,7 +49,7 @@ Modify:
 
 Header:
 
-`model_name,input_tokens,output_tokens,num_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size`
+`model_name,input_tokens,output_tokens,num_requests,num_warmup_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size`
 
 Each row defines one benchmark job.
 
@@ -66,39 +69,35 @@ The configuration table in `model_configs_example.csv` details each individual m
 - `num_requests`  
   Total number of requests sent.
 
-  ⚠️ The run may timeout before all requests are sent.  
-  Configure the `timeout` parameter in `benchmarking_bundles/config.yaml` accordingly.
+- `num_warmup_requests`  
+  Number of throwaway warm-up requests sent **before** the measured run. Set per
+  row. To disable, leave the cell blank or `0`.
+
+  Synthetic warm-ups match the test's concurrency level, while real-workload warm-ups ignore the pacing rate and just fire everything immediately.
 
 - `concurrent_requests`  
   Enables **synthetic workload benchmarking**.
-
-  - Used when testing batching-enabled models
-  - Typical values: `1`, `4`, `8`, `16`
   - If this value is set, `qps` is ignored
 
 - `qps`  
   Enables **real workload benchmarking** (queries per second).
-
   - Recommended values `< 10`
   - Ignored if `concurrent_requests` is set
 
 - `qps_distribution`  
   Wait-time distribution between requests.
-
   Supported values:
   - `constant` (default)
   - `uniform`
   - `exponential`
-
   Ignored if concurrent_requests is set
+
 - `multimodal_img_size`  
   Used only for multimodal models to include an image in the benchmark requests.
-
   Supported values:
   - `small` → 500×500 px
   - `medium` → 1000×1000 px
   - `large` → 2000×2000 px
-
   For non-multimodal models, leave it empty.
 
 > **Important:**  
@@ -132,7 +131,7 @@ This mode is useful when:
 
 ### 4. Run the benchmark
 
-Once the configuration files are set up, run the benchmark from the **root of the AISK repository**:
+Once the configuration files are set up, run the benchmark from the **root of the benchmarking_bundles module**:
 
 ```bash
 bash run_synthetic_perfomance_bundle_eval.sh
@@ -202,7 +201,7 @@ Each row in the Excel file corresponds to **one row in `model_configs_example.cs
 The benchmarking pipeline automatically estimates **batching behavior** by analyzing request timing patterns.
 
 - Requests with identical `server_ttft` are grouped together
-- Batch size is inferred as the next power of two of the group size
+- Batch size is inferred by snapping the group size **up** to the nearest value in the `batch_sizes` list configured in `config.yaml` (defaults to powers of two up to 128)
 
 Reported batching fields
 

benchmarking/benchmarking_bundles/config.yaml

@@ -1,7 +1,7 @@
-model_configs_path: '<PATH TO AISK REPO HERE>/benchmarking/benchmarking_bundles/model_configs_example.csv'
+model_configs_path: '<PATH TO AISK REPO HERE>/benchmarking_bundles/model_configs_example.csv'
 llm_api: 'sncloud'
-output_files_dir: '<PATH TO AISK REPO HERE>/benchmarking/data/bundle_tests/output_files'
-consolidated_results_dir: '<PATH TO AISK REPO HERE>/benchmarking/data/bundle_tests/consolidated_results'
+output_files_dir: '<PATH TO AISK REPO HERE>/data/bundle_tests/output_files'
+consolidated_results_dir: '<PATH TO AISK REPO HERE>/data/bundle_tests/consolidated_results'
 timeout: 3600
 time_delay: 0
 # Row-level concurrency

benchmarking/benchmarking_bundles/model_configs_example.csv

@@ -1,7 +1,7 @@
-model_name,input_tokens,output_tokens,num_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size
-Meta-Llama-3.3-70B-Instruct,128,128,1,1,,,
-Meta-Llama-3.3-70B-Instruct,128,128,2,2,,,
-Meta-Llama-3.3-70B-Instruct,128,128,4,4,,,
-Meta-Llama-3.3-70B-Instruct,128,128,8,8,,,
-Meta-Llama-3.3-70B-Instruct,128,128,16,16,,,
-Meta-Llama-3.3-70B-Instruct,128,128,32,32,,,
+model_name,input_tokens,output_tokens,num_requests,num_warmup_requests,concurrent_requests,qps,qps_distribution,multimodal_img_size
+Meta-Llama-3.3-70B-Instruct,128,128,1,1,1,,,
+Meta-Llama-3.3-70B-Instruct,128,128,2,2,2,,,
+Meta-Llama-3.3-70B-Instruct,128,128,4,4,4,,,
+Meta-Llama-3.3-70B-Instruct,128,128,8,8,8,,,
+Meta-Llama-3.3-70B-Instruct,128,128,16,16,16,,,
+Meta-Llama-3.3-70B-Instruct,128,128,32,32,32,,,

benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py

@@ -39,6 +39,7 @@ class ModelConfigRow:
     input_tokens: int
     output_tokens: int
     num_requests: int
+    num_warmup_requests: int = 0
     concurrent_requests: Optional[int] = None
     qps: Optional[float] = None
     qps_distribution: str = 'constant'
@@ -373,6 +374,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None:
 
         model_name = row['model_name']
         num_requests = int(row['num_requests'])
+        num_warmup_requests = int(row.get('num_warmup_requests', 0) or 0)
         input_tokens = int(row['input_tokens'])
         output_tokens = int(row['output_tokens'])
         concurrent_requests = int(row.get('concurrent_requests', 0) or 0)
@@ -391,6 +393,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None:
                     user_metadata={'model_idx': 0},
                     llm_api=self.config['llm_api'],
                     use_multiple_prompts=self.config['use_multiple_prompts'],
+                    num_warmup_requests=num_warmup_requests,
                 )
             elif qps:
                 evaluator = RealWorkLoadPerformanceEvaluator(
@@ -402,6 +405,7 @@ def _run_single_row(self, row: pd.Series, output_files_dir: str) -> None:
                     timeout=self.config['timeout'],
                     user_metadata={'model_idx': 0},
                     llm_api=self.config['llm_api'],
+                    num_warmup_requests=num_warmup_requests,
                 )
             else:
                 logger.warning(f'Skipping {model_name}: missing concurrency or QPS.')
@@ -423,9 +427,12 @@ def run(self, run_name: Optional[str] = None) -> None:
         from concurrent.futures import ThreadPoolExecutor, as_completed
 
         model_configs_df = pd.read_csv(self.config['model_configs_path'])
-        model_configs_df = model_configs_df.astype(
-            {'input_tokens': 'Int64', 'output_tokens': 'Int64', 'num_requests': 'Int64'}
-        )
+        int_columns = {'input_tokens': 'Int64', 'output_tokens': 'Int64', 'num_requests': 'Int64'}
+        # `num_warmup_requests` is an optional per-row column; only cast it when present so older
+        # CSVs without the column still load.
+        if 'num_warmup_requests' in model_configs_df.columns:
+            int_columns['num_warmup_requests'] = 'Int64'
+        model_configs_df = model_configs_df.astype(int_columns)
 
         run_time = datetime.now().strftime('%Y%m%d-%H%M%S.%f')
         if not run_name:

benchmarking/requirements.txt

@@ -27,6 +27,7 @@ transformers==4.57.1
 joblib==1.4.2
 openpyxl==3.1.5
 streamlit==1.51.0
-# vLLM (Optional - Linux only, requires NVIDIA GPU with CUDA)
-# Uncomment the line below if you're on Linux and want vLLM benchmarking support
+# vLLM — used only for the optional vLLM speed-benchmark comparison (benchmark-mode vllm/both).
+# For API benchmarking it drives a remote endpoint, so no local GPU is required.
+# Native install is best supported on Linux; on other platforms it may require extra steps.
 vllm>=0.6.0

benchmarking/run_custom_dataset.sh

@@ -5,10 +5,11 @@ ulimit -n 4096
 python src/evaluator.py \
 --mode custom \
 --model-name "Meta-Llama-3.3-70B-Instruct" \
---results-dir "./data/results/llmperf" \
+--results-dir "./data/results" \
 --num-concurrent-requests 1 \
 --timeout 600 \
---input-file-path "<AISK_REPOSITORY_PATH>/benchmarking/prompts/custom_prompt_example.jsonl" \
+--input-file-path "<AISK_REPOSITORY_PATH>/prompts/custom_prompt_example.jsonl" \
+--num-warmup-requests 0 \
 --save-llm-responses False \
 --sampling-params '{"max_tokens_to_generate": 256}' \
 --use-debugging-mode False \
@@ -24,10 +25,11 @@ python src/evaluator.py \
 # python src/evaluator.py \
 # --mode custom \
 # --model-name "Meta-Llama-3.3-70B-Instruct" \
-# --results-dir "./data/results/llmperf" \
+# --results-dir "./data/results" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \
 # --input-file-path "<CUSTOM DATASET PATH HERE>" \
+# --num-warmup-requests 0 \
 # --save-llm-responses False \
 # --sampling-params '{"max_tokens_to_generate": 256}' \
 # --use-debugging-mode False \
@@ -38,10 +40,11 @@ python src/evaluator.py \
 # python src/evaluator.py \
 # --mode custom \
 # --model-name "gemma-4-31B-it" \
-# --results-dir "./data/results/llmperf" \
+# --results-dir "./data/results" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \
 # --input-file-path "<CUSTOM DATASET PATH HERE>" \
+# --num-warmup-requests 0 \
 # --save-llm-responses False \
 # --sampling-params '{"max_tokens_to_generate": 256}' \
 # --use-debugging-mode False \

benchmarking/run_real_workload_dataset.sh

@@ -4,15 +4,16 @@
 ulimit -n 4096
 python src/evaluator.py \
 --mode real_workload \
---model-name "Meta-Llama-3.3-70B-Instruct" \
---results-dir "./data/results/llmperf" \
+--model-names "Meta-Llama-3.3-70B-Instruct" \
+--results-dir "./data/results" \
 --qps 1 \
 --qps-distribution "constant" \
 --timeout 600 \
 --num-input-tokens 1000 \
 --num-output-tokens 1000 \
 --multimodal-image-size na \
 --num-requests 16 \
+--num-warmup-requests 0 \
 --use-debugging-mode False \
 --llm-api sncloud
 
@@ -27,13 +28,14 @@ python src/evaluator.py \
 # python src/evaluator.py \
 # --mode real_workload \
 # --model-names "Meta-Llama-3.3-70B-Instruct" \
-# --results-dir "./data/results/llmperf" \
+# --results-dir "./data/results" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \
 # --num-input-tokens 1000 \
 # --num-output-tokens 1000 \
 # --multimodal-image-size na \
 # --num-requests 16 \
+# --num-warmup-requests 0 \
 # --use-debugging-mode False \
 # --llm-api sncloud
 
@@ -42,12 +44,13 @@ python src/evaluator.py \
 # python src/evaluator.py \
 # --mode real_workload \
 # --model-names "gemma-4-31B-it" \
-# --results-dir "./data/results/llmperf" \
+# --results-dir "./data/results" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \
 # --num-input-tokens 1000 \
 # --num-output-tokens 1000 \
 # --multimodal-image-size medium \
 # --num-requests 16 \
+# --num-warmup-requests 0 \
 # --use-debugging-mode False \
 # --llm-api sncloud

benchmarking/run_synthetic_dataset.sh

@@ -12,6 +12,7 @@ python src/evaluator.py \
 --num-output-tokens 1000 \
 --multimodal-image-size na \
 --num-requests 16 \
+--num-warmup-requests 0 \
 --use-multiple-prompts False \
 --save-llm-responses False \
 --use-debugging-mode False \
@@ -36,6 +37,7 @@ python src/evaluator.py \
 # --num-output-tokens 1000 \
 # --multimodal-image-size na \
 # --num-requests 16 \
+# --num-warmup-requests 0 \
 # --save-llm-responses False \
 # --use-debugging-mode False \
 # --llm-api sncloud
@@ -53,6 +55,7 @@ python src/evaluator.py \
 # --num-output-tokens 1000 \
 # --multimodal-image-size medium \
 # --num-requests 16 \
+# --num-warmup-requests 0 \
 # --save-llm-responses False \
 # --use-debugging-mode False \
 # --llm-api sncloud

benchmarking/src/evaluator.py

@@ -121,6 +121,16 @@ def main() -> None:
             (default: %(default)s)',
     )
 
+    parser.add_argument(
+        '--num-warmup-requests',
+        type=int,
+        required=False,
+        default=0,
+        help='Number of throwaway warm-up requests to send before the measured run. Warm-up requests are \
+            sent at the test concurrency and their results are discarded, absorbing cold-start and batch \
+            ramp-up costs so they do not skew the reported metrics. 0 disables warm-up. (default: %(default)s)',
+    )
+
     args, _ = parser.parse_known_args()
 
     # Parse user metadata.
@@ -174,6 +184,7 @@ def main() -> None:
             save_response_texts=args.save_llm_responses,
             use_debugging_mode=args.use_debugging_mode,
             llm_api=args.llm_api,
+            num_warmup_requests=args.num_warmup_requests,
         )
 
         # Run performance evaluation
@@ -278,6 +289,7 @@ def main() -> None:
                     save_response_texts=args.save_llm_responses,
                     use_debugging_mode=args.use_debugging_mode,
                     llm_api=args.llm_api,
+                    num_warmup_requests=args.num_warmup_requests,
                 )
 
                 # Run performance evaluation
@@ -300,6 +312,7 @@ def main() -> None:
                     results_dir=args.results_dir,
                     timeout=args.timeout,
                     user_metadata=user_metadata,
+                    num_warmup_requests=args.num_warmup_requests,
                 )
 
                 vllm_executor.run_benchmark(
@@ -397,6 +410,7 @@ def main() -> None:
                 user_metadata=user_metadata,
                 use_debugging_mode=args.use_debugging_mode,
                 llm_api=args.llm_api,
+                num_warmup_requests=args.num_warmup_requests,
             )
 
             # Run performance evaluation