R25.03 shantanu

.github/workflows/pre-commit.yaml → .github/workflows/pre-commit.yml

@@ -1,4 +1,4 @@
-# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -31,15 +31,15 @@ on:
 
 jobs:
   pre-commit:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v5.0.0
       with:
         fetch-depth: 2
     - name: Get modified files
       id: modified-files
       run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT
-    - uses: actions/setup-python@v3
-    - uses: pre-commit/action@v3.0.0
+    - uses: actions/setup-python@v6.0.0
+    - uses: pre-commit/action@v3.0.1
       with:
         extra_args: --files ${{ steps.modified-files.outputs.modified_files }}

.gitignore

@@ -10,8 +10,12 @@ test_results.txt
 artifacts
 cprofile
 *.prof
+.venv
+**/.venv
 
 # Test exclusions
 qa/L0_openai/openai
 tensorrtllm_models
+tensorrtllm_mistral_models/
 custom_tokenizer
+replace-artifacts/

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -25,7 +25,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 repos:
-- repo: https://github.com/timothycrosley/isort
+- repo: https://github.com/PyCQA/isort
   rev: 5.12.0
   hooks:
   - id: isort
@@ -36,7 +36,7 @@ repos:
   - id: black
     types_or: [python, cython]
 - repo: https://github.com/PyCQA/flake8
-  rev: 5.0.4
+  rev: 7.3.0
   hooks:
   - id: flake8
     args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501]
@@ -57,7 +57,7 @@ repos:
 # More details about these pre-commit hooks here:
 # https://pre-commit.com/hooks.html
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v6.0.0
   hooks:
   - id: check-case-conflict
   - id: check-executables-have-shebangs

CMakeLists.txt

@@ -26,6 +26,21 @@
 
 cmake_minimum_required(VERSION 3.18)
 
+# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5).
+# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a
+# separate CMake invocation; that step does not inherit -D variables from the top
+# level unless they are also in the environment. If configure fails inside libevent
+# with "Compatibility with CMake < 3.5 has been removed", use either:
+#   export CMAKE_POLICY_VERSION_MINIMUM=3.5   # before cmake AND cmake --build
+# or install/use CMake 3.28.x–3.31.x for this build.
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0")
+  message(
+    STATUS
+    "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export "
+    "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build "
+    "(see docs/customization_guide/build.md).")
+endif()
+
 project(tritonserver LANGUAGES C CXX)
 
 include(CMakeDependentOption)
@@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF)
 option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF)
 option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF)
 
+option(TRITON_ENABLE_MYSQL_ODBC
+  "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)"
+  OFF)
+
 # Need to know if TensorRT is available when building unit tests
 option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF)
 
@@ -261,6 +280,7 @@ ExternalProject_Add(triton-server
     -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3}
     -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT}
     -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE}
+    -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC}
     -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
     -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}

Dockerfile

@@ -0,0 +1,109 @@
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Derivative image: start from an official Triton server image and replace only
+# the tritonserver executable and libtritonserver.so. Backends, Python wheels,
+# and the rest of the filesystem stay unchanged from the base image.
+#
+# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX):
+#   install/bin/tritonserver
+#   install/lib/libtritonserver.so
+#
+# Prepare artifacts (from repository root):
+#   mkdir -p replace-artifacts
+#   cp install/bin/tritonserver replace-artifacts/
+#   cp install/lib/libtritonserver.so replace-artifacts/
+#   cp /etc/odbc.ini replace-artifacts/odbc.ini   # optional if databaseIp is set (DSN unused)
+#   Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for
+#   optional JSON field "odbcDriverName" if the default fails.
+#
+# Build (from repository root; match your Triton tag, e.g. r25.03):
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \
+#     -t tritonserver:25.03-custom .
+#
+# CPU-only base example:
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \
+#     -t tritonserver:25.03-custom-cpu .
+#
+# If your base image stores libtritonserver.so under lib64:
+#   --build-arg TRITON_LIB_SUBDIR=lib64
+#
+# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.:
+#   --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04
+#
+# Run (mount a model repository and optionally override DM config):
+#   docker run --rm --gpus=all \
+#     -p8000:8000 -p8001:8001 -p8002:8002 \
+#     -v /path/to/model_repo:/models:ro \
+#     -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \
+#     tritonserver:25.03-custom \
+#     tritonserver --model-repository=/models
+#
+# For CPU-only, drop --gpus=all and use a CPU/min base image.
+
+ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3
+
+FROM ${BASE_IMAGE}
+
+ARG TRITON_INSTALL_PREFIX=/opt/tritonserver
+ARG TRITON_LIB_SUBDIR=lib
+
+ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04
+ARG MYSQL_ODBC_DEB_ARCH=amd64
+
+# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe
+# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc.
+USER root
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unixodbc \
+    odbcinst; \
+  DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \
+  curl -fsSL -o "/tmp/${DEB}" \
+    "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \
+  apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \
+  rm -f "/tmp/${DEB}"; \
+  rm -rf /var/lib/apt/lists/*
+
+# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in
+# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under
+# /usr/lib/<triplet>/odbc/ only. unixODBC then fails with "Can't open lib
+# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/.
+RUN set -eux; \
+  mkdir -p /usr/lib/odbc; \
+  for f in \
+      /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \
+      /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \
+  do \
+    if [ -f "${f}" ]; then \
+      ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \
+    fi; \
+  done; \
+  test -f /usr/lib/odbc/libmyodbc9w.so
+
+# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json).
+COPY replace-artifacts/odbc.ini /etc/odbc.ini
+RUN chmod 644 /etc/odbc.ini
+
+# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro)
+COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json
+RUN chmod 644 /etc/triton-dmconfig.json
+
+# Paths relative to the build context (repository root when building with ".")
+COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver
+COPY replace-artifacts/libtritonserver.so \
+  ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so
+
+# Match ownership used by generated Triton Dockerfiles (triton-server uid)
+RUN chown 1000:1000 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \
+  && chmod 755 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so
+

Dockerfile.QA

@@ -61,6 +61,7 @@ RUN apt-get update && \
             python3-pip \
             python3-wheel \
             python3-setuptools \
+            python3-venv \
             rapidjson-dev \
             software-properties-common && \
     rm -rf /var/lib/apt/lists/*
@@ -74,12 +75,19 @@ RUN apt update -q=2 \
     && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3*
 
 # Add inception_graphdef model to example repo
+# FIXME: This should be changed to using the fetch_models.sh script
+# in order to ensure the public facing docs are up-to-date.
 WORKDIR /workspace/docs/examples/model_repository
-RUN mkdir -p inception_graphdef/1 && \
-    wget -O ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb.tar.gz \
-        https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \
-    (cd ${TRITONTMP_DIR} && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \
-    mv ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb inception_graphdef/1/model.graphdef
+RUN mkdir -p model_repository/inception_onnx/1 && \
+        wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \
+            https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \
+        (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \
+        python3 -m venv tf2onnx && \
+            source ./tf2onnx/bin/activate && \
+            pip3 install "numpy<2" tensorflow tf2onnx && \
+            python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 && \
+        deactivate  && \
+        mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx
 
 # Update the qa/ directory with test executables, models, etc.
 WORKDIR /workspace
@@ -109,7 +117,7 @@ RUN mkdir -p qa/common && \
     cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \
     cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \
     cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \
-    cp -r docs/examples/model_repository/inception_graphdef qa/L0_grpc/models && \
+    cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \
     mkdir qa/L0_grpc_state_cleanup/models && \
     cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \
     mkdir qa/L0_http/models && \
@@ -118,7 +126,7 @@ RUN mkdir -p qa/common && \
     cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \
     cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \
     cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \
-    cp -r docs/examples/model_repository/inception_graphdef qa/L0_http/models && \
+    cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \
     mkdir qa/L0_https/models && \
     cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \
     mkdir qa/L0_secure_grpc/models && \
@@ -149,21 +157,20 @@ RUN mkdir -p qa/common && \
     cp bin/triton_json_test qa/L0_json/. && \
     cp bin/backend_output_detail_test qa/L0_backend_output_detail/. && \
     cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/. && \
-    cp bin/input_byte_size_test qa/L0_input_validation/. && \
-    cp -r docs/examples/model_repository/simple_identity qa/L0_input_validation/models
+    cp bin/input_byte_size_test qa/L0_input_validation/.
 
 RUN mkdir -p qa/pkgs && \
     cp python/triton*.whl qa/pkgs/. && \
     cp -rf python/test/. qa/L0_python_api/.
 
 RUN mkdir -p qa/L0_simple_ensemble/models/simple/1 && \
-    cp docs/examples/model_repository/simple/1/model.graphdef \
+    cp docs/examples/model_repository/simple/1/model.onnx \
         qa/L0_simple_ensemble/models/simple/1/. && \
     mkdir -p qa/L0_simple_ensemble/models/simple/2 && \
-    cp docs/examples/model_repository/simple/1/model.graphdef \
+    cp docs/examples/model_repository/simple/1/model.onnx \
         qa/L0_simple_ensemble/models/simple/2/. && \
     mkdir -p qa/L0_socket/models/simple/1 && \
-    cp docs/examples/model_repository/simple/1/model.graphdef \
+    cp docs/examples/model_repository/simple/1/model.onnx \
         qa/L0_socket/models/simple/1/.
 
 RUN mkdir -p qa/L0_backend_identity/models && \

Dockerfile.pubmatic-bt-tritonserver

@@ -0,0 +1,52 @@
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Extend the image produced by build.py (`docker build -t tritonserver`) with
+# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files.
+# Build context: repository root (same as build.py final docker build).
+#
+# Prerequisites:
+#   - Image `tritonserver` must exist.
+#   - replace-artifacts/odbc.ini
+#   - replace-artifacts/triton-dmconfig.json
+#
+# Example:
+#   docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver .
+
+FROM tritonserver
+
+ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04
+ARG MYSQL_ODBC_DEB_ARCH=amd64
+
+USER root
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unixodbc \
+    odbcinst; \
+  DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \
+  curl -fsSL -o "/tmp/${DEB}" \
+    "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \
+  apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \
+  rm -f "/tmp/${DEB}"; \
+  rm -rf /var/lib/apt/lists/*
+
+RUN set -eux; \
+  mkdir -p /usr/lib/odbc; \
+  for f in \
+      /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \
+      /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \
+  do \
+    if [ -f "${f}" ]; then \
+      ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \
+    fi; \
+  done; \
+  test -f /usr/lib/odbc/libmyodbc9w.so
+
+COPY replace-artifacts/odbc.ini /etc/odbc.ini
+RUN chmod 644 /etc/odbc.ini
+
+COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json
+RUN chmod 644 /etc/triton-dmconfig.json

Dockerfile.sdk

@@ -29,7 +29,7 @@
 #
 
 # Base image on the minimum Triton container
-ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.02-py3-min
+ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min
 
 ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo
 ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo