Improvement: Make batch sizes configurable

benchmarking/benchmarking_bundles/config.yaml

@@ -8,4 +8,6 @@ time_delay: 0
 concurrency_enabled: False
 max_workers: 4 # Only used if concurrency_enabled is True
 # Prompt behavior
-use_multiple_prompts : False # Only available for synthetic performance evaluation
+use_multiple_prompts : False # Only available for synthetic performance evaluation
+# Batch sizes used to infer batching for the switching-time calculation.
+batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128] # add more numbers or non-power-of-two sizes (e.g. 6) when the deployment actually serves those batch sizes, so check your deployment configuration.

benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py

@@ -23,6 +23,10 @@
 )
 logger = logging.getLogger(__name__)
 
+# Default batch sizes used to infer batching (powers of two up to 128).
+# Can be overridden via the `batch_sizes` key in config.yaml.
+DEFAULT_BATCH_SIZES: List[int] = [1, 2, 4, 8, 16, 32, 64, 128]
+
 
 # =========================================================
 #                   DATA CLASSES
@@ -56,6 +60,7 @@ def load(self) -> Dict[str, Any]:
             cfg = yaml.load(fh, Loader=yaml.FullLoader)
         cfg['output_files_dir'] = os.path.expanduser(cfg.get('output_files_dir', '..'))
         cfg['model_configs_path'] = os.path.expanduser(cfg.get('model_configs_path', ''))
+        cfg['batch_sizes'] = cfg.get('batch_sizes') or DEFAULT_BATCH_SIZES
         return cfg
 
 
@@ -107,8 +112,25 @@ def extract_file_info(self, file_name: str) -> Tuple[str, int, int, Optional[int
 
 
 class BatchAnalyzer:
-    @staticmethod
-    def get_grouping_and_batching_info(df: pd.DataFrame) -> Tuple[List[int], List[int], pd.DataFrame]:
+    def __init__(self, batch_sizes: Optional[Sequence[int]] = None) -> None:
+        sizes = batch_sizes if batch_sizes else DEFAULT_BATCH_SIZES
+        self.batch_sizes = sorted({int(s) for s in sizes})
+
+    def _snap_batch_size(self, count: int) -> int:
+        """Snap an observed group count UP to the nearest allowed batch size.
+
+        Requests processed in the same server batch share an identical
+        server_ttft_s, so the count of consecutive identical-TTFT requests is
+        the observed group size. That count is mapped to the smallest configured
+        batch size that is >= it (falling back to the largest configured size if
+        the count exceeds every allowed value).
+        """
+        for size in self.batch_sizes:
+            if size >= count:
+                return size
+        return self.batch_sizes[-1]
+
+    def get_grouping_and_batching_info(self, df: pd.DataFrame) -> Tuple[List[int], List[int], pd.DataFrame]:
         if df.empty:
             return [], [], df
 
@@ -117,10 +139,10 @@ def get_grouping_and_batching_info(df: pd.DataFrame) -> Tuple[List[int], List[in
 
         group_counts = df.groupby(['group', 'server_ttft_s']).size().reset_index(name='consecutive_count')
         requests_grouping = group_counts['consecutive_count'].tolist()
-        requests_batching = [1 << (x - 1).bit_length() for x in requests_grouping]
+        requests_batching = [self._snap_batch_size(x) for x in requests_grouping]
 
         group_to_count = group_counts.set_index('group')['consecutive_count']
-        group_to_batching = {g: 1 << (cnt - 1).bit_length() for g, cnt in group_to_count.items()}
+        group_to_batching = {g: self._snap_batch_size(cnt) for g, cnt in group_to_count.items()}
 
         df['requests_grouping_per_request'] = df['group'].map(group_to_count)
         df['requests_batching_per_request'] = df['group'].map(group_to_batching)
@@ -140,10 +162,13 @@ def find_median_in_batches(lst: Sequence[int]) -> Optional[int]:
             return None
         total_sum = sum(lst)
         counter = Counter(lst)
-        for value, count in counter.items():
-            if (value * count) / total_sum > 0.5:
-                return value
-        return None
+        if total_sum > 0:
+            for value, count in counter.items():
+                if (value * count) / total_sum > 0.5:
+                    return value
+        # No single batch size dominates (e.g. a failed request skewed the
+        # distribution) - fall back to the most frequently observed batch size.
+        return counter.most_common(1)[0][0]
 
 
 # =========================================================
@@ -427,7 +452,7 @@ def run(self, run_name: Optional[str] = None) -> None:
 
         # Consolidation phase
         # For debugging, you can set a specific run_name here
-        # run_name = '20251031-180601.530151'
+        # run_name = '20260629-163957.994322'
         # output_files_dir = os.path.join(self.config['output_files_dir'], run_name)
         consolidator = ResultsConsolidator(
             self.read_perf_eval_json_files,
@@ -520,7 +545,7 @@ def main() -> None:
         evaluator_factories={},
         read_perf_eval_json_files_fn=read_perf_eval_json_files_wrapper,
         file_parser=FileNameParser(),
-        batch_analyzer=BatchAnalyzer(),
+        batch_analyzer=BatchAnalyzer(config.get('batch_sizes')),
         rep_finder=RepresentativeFinder(),
     )
 

benchmarking/run_custom_dataset.sh

@@ -37,7 +37,7 @@ python src/evaluator.py \
 
 # python src/evaluator.py \
 # --mode custom \
-# --model-name "gemma-3-12b-it" \
+# --model-name "gemma-4-31B-it" \
 # --results-dir "./data/results/llmperf" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \

benchmarking/run_real_workload_dataset.sh

@@ -41,7 +41,7 @@ python src/evaluator.py \
 
 # python src/evaluator.py \
 # --mode real_workload \
-# --model-names "gemma-3-12b-it" \
+# --model-names "gemma-4-31B-it" \
 # --results-dir "./data/results/llmperf" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \

benchmarking/run_synthetic_dataset.sh

@@ -45,7 +45,7 @@ python src/evaluator.py \
 
 # python src/evaluator.py \
 # --mode synthetic \
-# --model-names "gemma-3-12b-it" \
+# --model-names "gemma-4-31B-it" \
 # --results-dir "./data/results" \
 # --num-concurrent-requests 1 \
 # --timeout 600 \

enterprise_knowledge_retriever/streamlit/app.py

@@ -32,7 +32,7 @@
 # Available models in dropdown menu
 LLM_MODELS = [
     'gpt-oss-120b',
-    'gemma-3-12b-it',
+    'gemma-4-31B-it',
     'Meta-Llama-3.3-70B-Instruct',
     'DeepSeek-V3.1',
     'MiniMax-M2.5',

function_calling/src/function_calling.py

@@ -3,7 +3,7 @@
 import re
 import sys
 from pprint import pprint
-from typing import Any, Dict, List, Optional, Tuple, Type, Union
+from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast
 
 import yaml
 from dotenv import load_dotenv
@@ -14,7 +14,7 @@
 from langchain_core.messages.tool import ToolMessage
 from langchain_core.output_parsers import JsonOutputParser, StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.runnables import RunnableLambda
+from langchain_core.runnables import RunnableLambda, RunnableSequence
 from langchain_core.tools import StructuredTool, Tool
 from langchain_sambanova import ChatSambaNova
 from pydantic import BaseModel, Field, SecretStr
@@ -237,7 +237,7 @@ def get_tools_schemas(
 
         if default is not None:
             if isinstance(default, Tool) or isinstance(default, StructuredTool):
-                tool_schema = default.get_input_schema().model_json_schema()
+                tool_schema = cast(Type[BaseModel], default.get_input_schema()).model_json_schema()
             elif issubclass(default, BaseModel):
                 tool_schema = default.model_json_schema()
             else:
@@ -325,7 +325,9 @@ def function_call_llm(self, query: str, max_it: int = 10) -> str:
         tool_call_id = 0  # identification for each tool calling required to create ToolMessages
 
         for i in range(max_it):
-            json_parsing_chain = RunnableLambda(self.jsonFinder) | JsonOutputParser()
+            json_parsing_chain: RunnableSequence[BaseMessage, Any] = RunnableSequence(
+                RunnableLambda(self.jsonFinder), JsonOutputParser()
+            )
             print(f'\n\n---\nCalling function calling LLM with prompt: \n{history}\n')
             llm_response = self.llm.invoke(history)
             print(f'\nFunction calling LLM response: \n{llm_response}\n---\n')

multimodal_knowledge_retriever/config.yaml

@@ -6,7 +6,7 @@ llm:
     "model": "gpt-oss-120b"
 
 lvlm:
-    "model": "gemma-3-12b-it"
+    "model": "gemma-4-31B-it"
     "max_tokens": 8912 
     "temperature": 1
     "top_k": 50

multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb

@@ -70,7 +70,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -79,7 +79,7 @@
     "lvlm=ChatSambaNova(\n",
     "    api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n",
     "    base_url=\"https://api.sambanova.ai/v1\",\n",
-    "    model=\"gemma-3-12b-it\"\n",
+    "    model=\"gemma-4-31B-it\"\n",
     ")"
    ]
   },
@@ -1133,9 +1133,9 @@
       "  \"Male\": true,\n",
       "  \"Female\": false,\n",
       "  \"Type of injury\": \"grade 1 sprained right ankle\",\n",
-      "  \"Details of incident\": \"During emergency call #783B06, I slipped on the steps of the caller\u2019s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.\",\n",
+      "  \"Details of incident\": \"During emergency call #783B06, I slipped on the steps of the caller’s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.\",\n",
       "  \"Did the injury require a physician/hospital visit?\": \"Yes\",\n",
-      "  \"Name of physician/hospital\": \"St. Mary\u2019s Prompt Care\",\n",
+      "  \"Name of physician/hospital\": \"St. Mary’s Prompt Care\",\n",
       "  \"Physician/hospital phone number\": \"(708) 238-3222\",\n",
       "  \"Signature of injured party\": \"Carlos LaMachia\",\n",
       "  \"Date\": \"March 4, 2019\"\n",
@@ -1201,7 +1201,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'Date of incident': 'March 1, 2019', 'Time': '2:25 PM', 'Name of injured person': 'Carlos LaMachia, EMT', 'Address': '5621 Evergreen Ct./Rockview, IL 61233', 'Phone number(s)': '(630) 555-4455, cell', 'Age': '33', 'Male': True, 'Female': False, 'Type of injury': 'grade 1 sprained right ankle', 'Details of incident': 'During emergency call #783B06, I slipped on the steps of the caller\u2019s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.', 'Did the injury require a physician/hospital visit?': 'Yes', 'Name of physician/hospital': 'St. Mary\u2019s Prompt Care', 'Physician/hospital phone number': '(708) 238-3222', 'Signature of injured party': 'Carlos LaMachia', 'Date': 'March 4, 2019'}\n"
+      "{'Date of incident': 'March 1, 2019', 'Time': '2:25 PM', 'Name of injured person': 'Carlos LaMachia, EMT', 'Address': '5621 Evergreen Ct./Rockview, IL 61233', 'Phone number(s)': '(630) 555-4455, cell', 'Age': '33', 'Male': True, 'Female': False, 'Type of injury': 'grade 1 sprained right ankle', 'Details of incident': 'During emergency call #783B06, I slipped on the steps of the caller’s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.', 'Did the injury require a physician/hospital visit?': 'Yes', 'Name of physician/hospital': 'St. Mary’s Prompt Care', 'Physician/hospital phone number': '(708) 238-3222', 'Signature of injured party': 'Carlos LaMachia', 'Date': 'March 4, 2019'}\n"
      ]
     },
     {
@@ -1251,4 +1251,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb

@@ -52,7 +52,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -61,7 +61,7 @@
     "lvlm=ChatSambaNova(\n",
     "    api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n",
     "    base_url=\"https://api.sambanova.ai/v1\",\n",
-    "    model=\"gemma-3-12b-it\"\n",
+    "    model=\"gemma-4-31B-it\"\n",
     ")"
    ]
   },
@@ -1072,4 +1072,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

multimodal_knowledge_retriever/streamlit/app.py

@@ -33,12 +33,12 @@
 APP_DESCRIPTION_PATH = os.path.join(kit_dir, 'streamlit', 'app_description.yaml')
 # Available models in dropdown menu
 LVLM_MODELS = [
-    'gemma-3-12b-it',
+    'gemma-4-31B-it',
 ]
 # Available models in dropdown menu
 LLM_MODELS = [
     'gpt-oss-120b',
-    'gemma-3-12b-it',
+    'gemma-4-31B-it',
     'Meta-Llama-3.3-70B-Instruct',
     'DeepSeek-V3.1',
     'MiniMax-M2.5',

quickstart/multimodal_get_started.ipynb

@@ -81,7 +81,7 @@
     "lvlm=ChatSambaNova(\n",
     "    base_url = \"https://api.sambanova.ai/v1\",\n",
     "    api_key=os.environ.get('SAMBANOVA_API_KEY'),\n",
-    "    model=\"gemma-3-12b-it\", \n",
+    "    model=\"gemma-4-31B-it\", \n",
     "    temperature = 0.01\n",
     ")"
    ]
@@ -1004,4 +1004,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

search_assistant/streamlit/app.py

@@ -30,7 +30,7 @@
 # Available models in dropdown menu
 LLM_MODELS = [
     'gpt-oss-120b',
-    'gemma-3-12b-it',
+    'gemma-4-31B-it',
     'Meta-Llama-3.3-70B-Instruct',
     'DeepSeek-V3.1',
     'MiniMax-M2.5',

utils/connection_pooling/README.md

@@ -35,7 +35,7 @@ python3 test_connection_pooling.py --api-url <URL> --api-key <KEY> --model <MODE
 python3 test_connection_pooling.py \
   --api-url https://api.sambanova.ai/v1/chat/completions \
   --api-key your-api-key-here \
-  --model gemma-3-12b-it \
+  --model gemma-4-31B-it \
   --test-duration 100 \
   --sleep-duration 20 \
   --max-tokens 100

utils/tests/config.yaml

@@ -1,10 +1,10 @@
 models:
   text:
-    - "gemma-3-12b-it"
+    - "gemma-4-31B-it"
     - "Meta-Llama-3.3-70B-Instruct"
 
   text-image:
-    - "gemma-3-12b-it"
+    - "gemma-4-31B-it"
 
   text-audio:
     - "Whisper-Large-v3"