From cbaaac038cdd1b47bb38e274fdcffe5bc6ecde63 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Thu, 2 Jul 2026 12:27:07 -0500 Subject: [PATCH 1/7] made batch sizes configurable and fixed representative batch size calculation when there are failed requests --- .../synthetic_performance_eval_script.py | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py b/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py index 715667079..e5d37404d 100644 --- a/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py +++ b/benchmarking/benchmarking_bundles/synthetic_performance_eval_script.py @@ -23,6 +23,10 @@ ) logger = logging.getLogger(__name__) +# Default batch sizes used to infer batching (powers of two up to 128). +# Can be overridden via the `batch_sizes` key in config.yaml. +DEFAULT_BATCH_SIZES: List[int] = [1, 2, 4, 8, 16, 32, 64, 128] + # ========================================================= # DATA CLASSES @@ -56,6 +60,7 @@ def load(self) -> Dict[str, Any]: cfg = yaml.load(fh, Loader=yaml.FullLoader) cfg['output_files_dir'] = os.path.expanduser(cfg.get('output_files_dir', '..')) cfg['model_configs_path'] = os.path.expanduser(cfg.get('model_configs_path', '')) + cfg['batch_sizes'] = cfg.get('batch_sizes') or DEFAULT_BATCH_SIZES return cfg @@ -107,8 +112,25 @@ def extract_file_info(self, file_name: str) -> Tuple[str, int, int, Optional[int class BatchAnalyzer: - @staticmethod - def get_grouping_and_batching_info(df: pd.DataFrame) -> Tuple[List[int], List[int], pd.DataFrame]: + def __init__(self, batch_sizes: Optional[Sequence[int]] = None) -> None: + sizes = batch_sizes if batch_sizes else DEFAULT_BATCH_SIZES + self.batch_sizes = sorted({int(s) for s in sizes}) + + def _snap_batch_size(self, count: int) -> int: + """Snap an observed group count UP to the nearest allowed batch size. + + Requests processed in the same server batch share an identical + server_ttft_s, so the count of consecutive identical-TTFT requests is + the observed group size. That count is mapped to the smallest configured + batch size that is >= it (falling back to the largest configured size if + the count exceeds every allowed value). + """ + for size in self.batch_sizes: + if size >= count: + return size + return self.batch_sizes[-1] + + def get_grouping_and_batching_info(self, df: pd.DataFrame) -> Tuple[List[int], List[int], pd.DataFrame]: if df.empty: return [], [], df @@ -117,10 +139,10 @@ def get_grouping_and_batching_info(df: pd.DataFrame) -> Tuple[List[int], List[in group_counts = df.groupby(['group', 'server_ttft_s']).size().reset_index(name='consecutive_count') requests_grouping = group_counts['consecutive_count'].tolist() - requests_batching = [1 << (x - 1).bit_length() for x in requests_grouping] + requests_batching = [self._snap_batch_size(x) for x in requests_grouping] group_to_count = group_counts.set_index('group')['consecutive_count'] - group_to_batching = {g: 1 << (cnt - 1).bit_length() for g, cnt in group_to_count.items()} + group_to_batching = {g: self._snap_batch_size(cnt) for g, cnt in group_to_count.items()} df['requests_grouping_per_request'] = df['group'].map(group_to_count) df['requests_batching_per_request'] = df['group'].map(group_to_batching) @@ -140,10 +162,13 @@ def find_median_in_batches(lst: Sequence[int]) -> Optional[int]: return None total_sum = sum(lst) counter = Counter(lst) - for value, count in counter.items(): - if (value * count) / total_sum > 0.5: - return value - return None + if total_sum > 0: + for value, count in counter.items(): + if (value * count) / total_sum > 0.5: + return value + # No single batch size dominates (e.g. a failed request skewed the + # distribution) - fall back to the most frequently observed batch size. + return counter.most_common(1)[0][0] # ========================================================= @@ -427,7 +452,7 @@ def run(self, run_name: Optional[str] = None) -> None: # Consolidation phase # For debugging, you can set a specific run_name here - # run_name = '20251031-180601.530151' + # run_name = '20260629-163957.994322' # output_files_dir = os.path.join(self.config['output_files_dir'], run_name) consolidator = ResultsConsolidator( self.read_perf_eval_json_files, @@ -520,7 +545,7 @@ def main() -> None: evaluator_factories={}, read_perf_eval_json_files_fn=read_perf_eval_json_files_wrapper, file_parser=FileNameParser(), - batch_analyzer=BatchAnalyzer(), + batch_analyzer=BatchAnalyzer(config.get('batch_sizes')), rep_finder=RepresentativeFinder(), ) From 198277230f17072a9cbd406fdd15389bdc80392e Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Thu, 2 Jul 2026 12:27:31 -0500 Subject: [PATCH 2/7] modified config for batch size configuration --- benchmarking/benchmarking_bundles/config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarking/benchmarking_bundles/config.yaml b/benchmarking/benchmarking_bundles/config.yaml index 0ffa2bbb3..1f29837b1 100644 --- a/benchmarking/benchmarking_bundles/config.yaml +++ b/benchmarking/benchmarking_bundles/config.yaml @@ -8,4 +8,6 @@ time_delay: 0 concurrency_enabled: False max_workers: 4 # Only used if concurrency_enabled is True # Prompt behavior -use_multiple_prompts : False # Only available for synthetic performance evaluation \ No newline at end of file +use_multiple_prompts : False # Only available for synthetic performance evaluation +# Batch sizes used to infer batching for the switching-time calculation. +batch_sizes: [1, 2, 4, 8, 16, 32, 64, 128] # add more numbers or non-power-of-two sizes (e.g. 6) when the deployment actually serves those batch sizes, so check your deployment configuration. \ No newline at end of file From cff17d11bcd699535a3cae5cfad5db88d256365d Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Thu, 2 Jul 2026 17:52:46 -0500 Subject: [PATCH 3/7] mypy fix --- function_calling/src/function_calling.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/function_calling/src/function_calling.py b/function_calling/src/function_calling.py index 836d4386f..928d41193 100644 --- a/function_calling/src/function_calling.py +++ b/function_calling/src/function_calling.py @@ -3,7 +3,7 @@ import re import sys from pprint import pprint -from typing import Any, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast import yaml from dotenv import load_dotenv @@ -14,7 +14,7 @@ from langchain_core.messages.tool import ToolMessage from langchain_core.output_parsers import JsonOutputParser, StrOutputParser from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnableLambda +from langchain_core.runnables import RunnableLambda, RunnableSequence from langchain_core.tools import StructuredTool, Tool from langchain_sambanova import ChatSambaNova from pydantic import BaseModel, Field, SecretStr @@ -237,7 +237,7 @@ def get_tools_schemas( if default is not None: if isinstance(default, Tool) or isinstance(default, StructuredTool): - tool_schema = default.get_input_schema().model_json_schema() + tool_schema = cast(Type[BaseModel], default.get_input_schema()).model_json_schema() elif issubclass(default, BaseModel): tool_schema = default.model_json_schema() else: @@ -325,7 +325,9 @@ def function_call_llm(self, query: str, max_it: int = 10) -> str: tool_call_id = 0 # identification for each tool calling required to create ToolMessages for i in range(max_it): - json_parsing_chain = RunnableLambda(self.jsonFinder) | JsonOutputParser() + json_parsing_chain: RunnableSequence[BaseMessage, Any] = RunnableSequence( + RunnableLambda(self.jsonFinder), JsonOutputParser() + ) print(f'\n\n---\nCalling function calling LLM with prompt: \n{history}\n') llm_response = self.llm.invoke(history) print(f'\nFunction calling LLM response: \n{llm_response}\n---\n') From 5f224013ac89c1b2c01e9c64d4c971b6d7fe8930 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Mon, 6 Jul 2026 12:35:00 -0500 Subject: [PATCH 4/7] gemma changes to version 4 --- benchmarking/run_custom_dataset.sh | 2 +- benchmarking/run_real_workload_dataset.sh | 2 +- benchmarking/run_synthetic_dataset.sh | 2 +- enterprise_knowledge_retriever/streamlit/app.py | 2 +- multimodal_knowledge_retriever/config.yaml | 2 +- .../notebooks/1_multimodal_get_started.ipynb | 12 ++++++------ .../notebooks/2_multimodal_rag.ipynb | 6 +++--- multimodal_knowledge_retriever/streamlit/app.py | 4 ++-- quickstart/multimodal_get_started.ipynb | 4 ++-- search_assistant/streamlit/app.py | 2 +- utils/connection_pooling/README.md | 2 +- utils/tests/config.yaml | 4 ++-- 12 files changed, 22 insertions(+), 22 deletions(-) diff --git a/benchmarking/run_custom_dataset.sh b/benchmarking/run_custom_dataset.sh index 2aa16108c..791e045cf 100644 --- a/benchmarking/run_custom_dataset.sh +++ b/benchmarking/run_custom_dataset.sh @@ -37,7 +37,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode custom \ -# --model-name "gemma-3-12b-it" \ +# --model-name "gemma-4-31b-it" \ # --results-dir "./data/results/llmperf" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/benchmarking/run_real_workload_dataset.sh b/benchmarking/run_real_workload_dataset.sh index 0bf42d90c..70a2b6cd6 100755 --- a/benchmarking/run_real_workload_dataset.sh +++ b/benchmarking/run_real_workload_dataset.sh @@ -41,7 +41,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode real_workload \ -# --model-names "gemma-3-12b-it" \ +# --model-names "gemma-4-31b-it" \ # --results-dir "./data/results/llmperf" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/benchmarking/run_synthetic_dataset.sh b/benchmarking/run_synthetic_dataset.sh index a96d1a80e..c5ee827d8 100755 --- a/benchmarking/run_synthetic_dataset.sh +++ b/benchmarking/run_synthetic_dataset.sh @@ -45,7 +45,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode synthetic \ -# --model-names "gemma-3-12b-it" \ +# --model-names "gemma-4-31b-it" \ # --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/enterprise_knowledge_retriever/streamlit/app.py b/enterprise_knowledge_retriever/streamlit/app.py index a351e4c5c..a16eda39f 100644 --- a/enterprise_knowledge_retriever/streamlit/app.py +++ b/enterprise_knowledge_retriever/streamlit/app.py @@ -32,7 +32,7 @@ # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-3-12b-it', + 'gemma-4-31b-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/multimodal_knowledge_retriever/config.yaml b/multimodal_knowledge_retriever/config.yaml index fbdc4e0a1..399cad786 100644 --- a/multimodal_knowledge_retriever/config.yaml +++ b/multimodal_knowledge_retriever/config.yaml @@ -6,7 +6,7 @@ llm: "model": "gpt-oss-120b" lvlm: - "model": "gemma-3-12b-it" + "model": "gemma-4-31b-it" "max_tokens": 8912 "temperature": 1 "top_k": 50 diff --git a/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb b/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb index d2c24d394..b6418b05c 100644 --- a/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb +++ b/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +79,7 @@ "lvlm=ChatSambaNova(\n", " api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n", " base_url=\"https://api.sambanova.ai/v1\",\n", - " model=\"gemma-3-12b-it\"\n", + " model=\"gemma-4-31b-it\"\n", ")" ] }, @@ -1133,9 +1133,9 @@ " \"Male\": true,\n", " \"Female\": false,\n", " \"Type of injury\": \"grade 1 sprained right ankle\",\n", - " \"Details of incident\": \"During emergency call #783B06, I slipped on the steps of the caller\u2019s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.\",\n", + " \"Details of incident\": \"During emergency call #783B06, I slipped on the steps of the caller’s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.\",\n", " \"Did the injury require a physician/hospital visit?\": \"Yes\",\n", - " \"Name of physician/hospital\": \"St. Mary\u2019s Prompt Care\",\n", + " \"Name of physician/hospital\": \"St. Mary’s Prompt Care\",\n", " \"Physician/hospital phone number\": \"(708) 238-3222\",\n", " \"Signature of injured party\": \"Carlos LaMachia\",\n", " \"Date\": \"March 4, 2019\"\n", @@ -1201,7 +1201,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'Date of incident': 'March 1, 2019', 'Time': '2:25 PM', 'Name of injured person': 'Carlos LaMachia, EMT', 'Address': '5621 Evergreen Ct./Rockview, IL 61233', 'Phone number(s)': '(630) 555-4455, cell', 'Age': '33', 'Male': True, 'Female': False, 'Type of injury': 'grade 1 sprained right ankle', 'Details of incident': 'During emergency call #783B06, I slipped on the steps of the caller\u2019s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.', 'Did the injury require a physician/hospital visit?': 'Yes', 'Name of physician/hospital': 'St. Mary\u2019s Prompt Care', 'Physician/hospital phone number': '(708) 238-3222', 'Signature of injured party': 'Carlos LaMachia', 'Date': 'March 4, 2019'}\n" + "{'Date of incident': 'March 1, 2019', 'Time': '2:25 PM', 'Name of injured person': 'Carlos LaMachia, EMT', 'Address': '5621 Evergreen Ct./Rockview, IL 61233', 'Phone number(s)': '(630) 555-4455, cell', 'Age': '33', 'Male': True, 'Female': False, 'Type of injury': 'grade 1 sprained right ankle', 'Details of incident': 'During emergency call #783B06, I slipped on the steps of the caller’s front porch, twisting my right ankle. I kept walking on the foot to complete the initial patient interview and checks. But I had to call a backup (John Baumgard) to assist with the safe transfer of the patient. After delivery to hospital was complete, I had my ankle checked and x-rayed.', 'Did the injury require a physician/hospital visit?': 'Yes', 'Name of physician/hospital': 'St. Mary’s Prompt Care', 'Physician/hospital phone number': '(708) 238-3222', 'Signature of injured party': 'Carlos LaMachia', 'Date': 'March 4, 2019'}\n" ] }, { @@ -1251,4 +1251,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb b/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb index f3c0dbe3e..e3d504828 100644 --- a/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb +++ b/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb @@ -52,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -61,7 +61,7 @@ "lvlm=ChatSambaNova(\n", " api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n", " base_url=\"https://api.sambanova.ai/v1\",\n", - " model=\"gemma-3-12b-it\"\n", + " model=\"gemma-4-31b-it\"\n", ")" ] }, @@ -1072,4 +1072,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/multimodal_knowledge_retriever/streamlit/app.py b/multimodal_knowledge_retriever/streamlit/app.py index bc1b82eae..fc0f8a970 100644 --- a/multimodal_knowledge_retriever/streamlit/app.py +++ b/multimodal_knowledge_retriever/streamlit/app.py @@ -33,12 +33,12 @@ APP_DESCRIPTION_PATH = os.path.join(kit_dir, 'streamlit', 'app_description.yaml') # Available models in dropdown menu LVLM_MODELS = [ - 'gemma-3-12b-it', + 'gemma-4-31b-it', ] # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-3-12b-it', + 'gemma-4-31b-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/quickstart/multimodal_get_started.ipynb b/quickstart/multimodal_get_started.ipynb index a11809c9b..6dd17a644 100644 --- a/quickstart/multimodal_get_started.ipynb +++ b/quickstart/multimodal_get_started.ipynb @@ -81,7 +81,7 @@ "lvlm=ChatSambaNova(\n", " base_url = \"https://api.sambanova.ai/v1\",\n", " api_key=os.environ.get('SAMBANOVA_API_KEY'),\n", - " model=\"gemma-3-12b-it\", \n", + " model=\"gemma-4-31b-it\", \n", " temperature = 0.01\n", ")" ] @@ -1004,4 +1004,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/search_assistant/streamlit/app.py b/search_assistant/streamlit/app.py index 6eb407fd6..34eebbc4d 100644 --- a/search_assistant/streamlit/app.py +++ b/search_assistant/streamlit/app.py @@ -30,7 +30,7 @@ # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-3-12b-it', + 'gemma-4-31b-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/utils/connection_pooling/README.md b/utils/connection_pooling/README.md index ae55f63e3..c9f9e4802 100644 --- a/utils/connection_pooling/README.md +++ b/utils/connection_pooling/README.md @@ -35,7 +35,7 @@ python3 test_connection_pooling.py --api-url --api-key --model Date: Mon, 6 Jul 2026 13:11:09 -0500 Subject: [PATCH 5/7] testing img and gemma --- utils/tests/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/tests/config.yaml b/utils/tests/config.yaml index 592a5a291..f2cd44a80 100644 --- a/utils/tests/config.yaml +++ b/utils/tests/config.yaml @@ -1,6 +1,6 @@ models: text: - - "gemma-4-31b-it" + # - "gemma-4-31b-it" - "Meta-Llama-3.3-70B-Instruct" text-image: From 33589daa0198e1463aba1523aff140273026b896 Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Mon, 6 Jul 2026 13:17:01 -0500 Subject: [PATCH 6/7] fixed gemma case --- utils/tests/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/tests/config.yaml b/utils/tests/config.yaml index f2cd44a80..ef6106024 100644 --- a/utils/tests/config.yaml +++ b/utils/tests/config.yaml @@ -1,10 +1,10 @@ models: text: - # - "gemma-4-31b-it" + - "gemma-4-31B-it" - "Meta-Llama-3.3-70B-Instruct" text-image: - - "gemma-4-31b-it" + - "gemma-4-31B-it" text-audio: - "Whisper-Large-v3" From a9a5e7b4dc8c73320f0a2c3d707ad632c5b7d7db Mon Sep 17 00:00:00 2001 From: Rodrigo Maldonado Date: Mon, 6 Jul 2026 13:32:12 -0500 Subject: [PATCH 7/7] fixed gemma casing --- benchmarking/run_custom_dataset.sh | 2 +- benchmarking/run_real_workload_dataset.sh | 2 +- benchmarking/run_synthetic_dataset.sh | 2 +- enterprise_knowledge_retriever/streamlit/app.py | 2 +- multimodal_knowledge_retriever/config.yaml | 2 +- .../notebooks/1_multimodal_get_started.ipynb | 2 +- .../notebooks/2_multimodal_rag.ipynb | 2 +- multimodal_knowledge_retriever/streamlit/app.py | 4 ++-- quickstart/multimodal_get_started.ipynb | 2 +- search_assistant/streamlit/app.py | 2 +- utils/connection_pooling/README.md | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/benchmarking/run_custom_dataset.sh b/benchmarking/run_custom_dataset.sh index 791e045cf..3810830b0 100644 --- a/benchmarking/run_custom_dataset.sh +++ b/benchmarking/run_custom_dataset.sh @@ -37,7 +37,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode custom \ -# --model-name "gemma-4-31b-it" \ +# --model-name "gemma-4-31B-it" \ # --results-dir "./data/results/llmperf" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/benchmarking/run_real_workload_dataset.sh b/benchmarking/run_real_workload_dataset.sh index 70a2b6cd6..6dfc840d3 100755 --- a/benchmarking/run_real_workload_dataset.sh +++ b/benchmarking/run_real_workload_dataset.sh @@ -41,7 +41,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode real_workload \ -# --model-names "gemma-4-31b-it" \ +# --model-names "gemma-4-31B-it" \ # --results-dir "./data/results/llmperf" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/benchmarking/run_synthetic_dataset.sh b/benchmarking/run_synthetic_dataset.sh index c5ee827d8..31e11c8eb 100755 --- a/benchmarking/run_synthetic_dataset.sh +++ b/benchmarking/run_synthetic_dataset.sh @@ -45,7 +45,7 @@ python src/evaluator.py \ # python src/evaluator.py \ # --mode synthetic \ -# --model-names "gemma-4-31b-it" \ +# --model-names "gemma-4-31B-it" \ # --results-dir "./data/results" \ # --num-concurrent-requests 1 \ # --timeout 600 \ diff --git a/enterprise_knowledge_retriever/streamlit/app.py b/enterprise_knowledge_retriever/streamlit/app.py index a16eda39f..2260ce389 100644 --- a/enterprise_knowledge_retriever/streamlit/app.py +++ b/enterprise_knowledge_retriever/streamlit/app.py @@ -32,7 +32,7 @@ # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-4-31b-it', + 'gemma-4-31B-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/multimodal_knowledge_retriever/config.yaml b/multimodal_knowledge_retriever/config.yaml index 399cad786..08abd95f6 100644 --- a/multimodal_knowledge_retriever/config.yaml +++ b/multimodal_knowledge_retriever/config.yaml @@ -6,7 +6,7 @@ llm: "model": "gpt-oss-120b" lvlm: - "model": "gemma-4-31b-it" + "model": "gemma-4-31B-it" "max_tokens": 8912 "temperature": 1 "top_k": 50 diff --git a/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb b/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb index b6418b05c..3195aad86 100644 --- a/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb +++ b/multimodal_knowledge_retriever/notebooks/1_multimodal_get_started.ipynb @@ -79,7 +79,7 @@ "lvlm=ChatSambaNova(\n", " api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n", " base_url=\"https://api.sambanova.ai/v1\",\n", - " model=\"gemma-4-31b-it\"\n", + " model=\"gemma-4-31B-it\"\n", ")" ] }, diff --git a/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb b/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb index e3d504828..88d3b1a5c 100644 --- a/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb +++ b/multimodal_knowledge_retriever/notebooks/2_multimodal_rag.ipynb @@ -61,7 +61,7 @@ "lvlm=ChatSambaNova(\n", " api_key=os.environ.get(\"SAMBANOVA_API_KEY\"),\n", " base_url=\"https://api.sambanova.ai/v1\",\n", - " model=\"gemma-4-31b-it\"\n", + " model=\"gemma-4-31B-it\"\n", ")" ] }, diff --git a/multimodal_knowledge_retriever/streamlit/app.py b/multimodal_knowledge_retriever/streamlit/app.py index fc0f8a970..a10176a4b 100644 --- a/multimodal_knowledge_retriever/streamlit/app.py +++ b/multimodal_knowledge_retriever/streamlit/app.py @@ -33,12 +33,12 @@ APP_DESCRIPTION_PATH = os.path.join(kit_dir, 'streamlit', 'app_description.yaml') # Available models in dropdown menu LVLM_MODELS = [ - 'gemma-4-31b-it', + 'gemma-4-31B-it', ] # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-4-31b-it', + 'gemma-4-31B-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/quickstart/multimodal_get_started.ipynb b/quickstart/multimodal_get_started.ipynb index 6dd17a644..ad0351f6e 100644 --- a/quickstart/multimodal_get_started.ipynb +++ b/quickstart/multimodal_get_started.ipynb @@ -81,7 +81,7 @@ "lvlm=ChatSambaNova(\n", " base_url = \"https://api.sambanova.ai/v1\",\n", " api_key=os.environ.get('SAMBANOVA_API_KEY'),\n", - " model=\"gemma-4-31b-it\", \n", + " model=\"gemma-4-31B-it\", \n", " temperature = 0.01\n", ")" ] diff --git a/search_assistant/streamlit/app.py b/search_assistant/streamlit/app.py index 34eebbc4d..9cd1d4ef5 100644 --- a/search_assistant/streamlit/app.py +++ b/search_assistant/streamlit/app.py @@ -30,7 +30,7 @@ # Available models in dropdown menu LLM_MODELS = [ 'gpt-oss-120b', - 'gemma-4-31b-it', + 'gemma-4-31B-it', 'Meta-Llama-3.3-70B-Instruct', 'DeepSeek-V3.1', 'MiniMax-M2.5', diff --git a/utils/connection_pooling/README.md b/utils/connection_pooling/README.md index c9f9e4802..48d3773e5 100644 --- a/utils/connection_pooling/README.md +++ b/utils/connection_pooling/README.md @@ -35,7 +35,7 @@ python3 test_connection_pooling.py --api-url --api-key --model