R25.03 shantanu

.gitignore

@@ -18,3 +18,4 @@ qa/L0_openai/openai
 tensorrtllm_models
 tensorrtllm_mistral_models/
 custom_tokenizer
+replace-artifacts/

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -24,7 +24,22 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cmake_minimum_required(VERSION 3.31.8)
+cmake_minimum_required(VERSION 3.18)
+
+# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5).
+# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a
+# separate CMake invocation; that step does not inherit -D variables from the top
+# level unless they are also in the environment. If configure fails inside libevent
+# with "Compatibility with CMake < 3.5 has been removed", use either:
+#   export CMAKE_POLICY_VERSION_MINIMUM=3.5   # before cmake AND cmake --build
+# or install/use CMake 3.28.x–3.31.x for this build.
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0")
+  message(
+    STATUS
+    "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export "
+    "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build "
+    "(see docs/customization_guide/build.md).")
+endif()
 
 project(tritonserver LANGUAGES C CXX)
 
@@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF)
 option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF)
 option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF)
 
+option(TRITON_ENABLE_MYSQL_ODBC
+  "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)"
+  OFF)
+
 # Need to know if TensorRT is available when building unit tests
 option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF)
 
@@ -166,9 +185,22 @@ if (OPENSSL_ROOT_DIR)
   set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}")
 endif()
 
-set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf")
+# Location where protobuf-config.cmake will be installed varies by
+# platform
+if (WIN32)
+  set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake")
+else()
+  set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf")
+endif()
 
-set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp")
+# Triton with Opentelemetry is not supported on Windows
+# FIXME: add location for Windows, when support is added
+# JIRA DLIS-4786
+if (WIN32)
+  set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "")
+else()
+  set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp")
+endif()
 
 if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
   set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install)
@@ -189,7 +221,7 @@ endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || TRITON_ENABLE_SAGEMAKER
 if(${TRITON_ENABLE_GRPC})
   set(TRITON_DEPENDS ${TRITON_DEPENDS} grpc)
 endif() # TRITON_ENABLE_GRPC
-if(${TRITON_ENABLE_TRACING})
+if(NOT WIN32 AND ${TRITON_ENABLE_TRACING})
   set(TRITON_DEPENDS ${TRITON_DEPENDS} opentelemetry-cpp)
 endif() # TRITON_ENABLE_TRACING
 
@@ -248,6 +280,7 @@ ExternalProject_Add(triton-server
     -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3}
     -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT}
     -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE}
+    -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC}
     -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
     -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}

CONTRIBUTING.md

@@ -88,7 +88,7 @@ proposed change so that the Triton team can provide feedback.
     documentation for instructions on running these tests.
 
 - Triton Inference Server's default build assumes recent versions of
-  dependencies (CUDA, PyTorch, TensorRT,
+  dependencies (CUDA, TensorFlow, PyTorch, TensorRT,
   etc.). Contributions that add compatibility with older versions of
   those dependencies will be considered, but NVIDIA cannot guarantee
   that all possible build configurations work, are not broken by

Dockerfile

@@ -0,0 +1,109 @@
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Derivative image: start from an official Triton server image and replace only
+# the tritonserver executable and libtritonserver.so. Backends, Python wheels,
+# and the rest of the filesystem stay unchanged from the base image.
+#
+# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX):
+#   install/bin/tritonserver
+#   install/lib/libtritonserver.so
+#
+# Prepare artifacts (from repository root):
+#   mkdir -p replace-artifacts
+#   cp install/bin/tritonserver replace-artifacts/
+#   cp install/lib/libtritonserver.so replace-artifacts/
+#   cp /etc/odbc.ini replace-artifacts/odbc.ini   # optional if databaseIp is set (DSN unused)
+#   Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for
+#   optional JSON field "odbcDriverName" if the default fails.
+#
+# Build (from repository root; match your Triton tag, e.g. r25.03):
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \
+#     -t tritonserver:25.03-custom .
+#
+# CPU-only base example:
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \
+#     -t tritonserver:25.03-custom-cpu .
+#
+# If your base image stores libtritonserver.so under lib64:
+#   --build-arg TRITON_LIB_SUBDIR=lib64
+#
+# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.:
+#   --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04
+#
+# Run (mount a model repository and optionally override DM config):
+#   docker run --rm --gpus=all \
+#     -p8000:8000 -p8001:8001 -p8002:8002 \
+#     -v /path/to/model_repo:/models:ro \
+#     -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \
+#     tritonserver:25.03-custom \
+#     tritonserver --model-repository=/models
+#
+# For CPU-only, drop --gpus=all and use a CPU/min base image.
+
+ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3
+
+FROM ${BASE_IMAGE}
+
+ARG TRITON_INSTALL_PREFIX=/opt/tritonserver
+ARG TRITON_LIB_SUBDIR=lib
+
+ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04
+ARG MYSQL_ODBC_DEB_ARCH=amd64
+
+# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe
+# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc.
+USER root
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unixodbc \
+    odbcinst; \
+  DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \
+  curl -fsSL -o "/tmp/${DEB}" \
+    "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \
+  apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \
+  rm -f "/tmp/${DEB}"; \
+  rm -rf /var/lib/apt/lists/*
+
+# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in
+# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under
+# /usr/lib/<triplet>/odbc/ only. unixODBC then fails with "Can't open lib
+# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/.
+RUN set -eux; \
+  mkdir -p /usr/lib/odbc; \
+  for f in \
+      /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \
+      /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \
+  do \
+    if [ -f "${f}" ]; then \
+      ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \
+    fi; \
+  done; \
+  test -f /usr/lib/odbc/libmyodbc9w.so
+
+# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json).
+COPY replace-artifacts/odbc.ini /etc/odbc.ini
+RUN chmod 644 /etc/odbc.ini
+
+# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro)
+COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json
+RUN chmod 644 /etc/triton-dmconfig.json
+
+# Paths relative to the build context (repository root when building with ".")
+COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver
+COPY replace-artifacts/libtritonserver.so \
+  ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so
+
+# Match ownership used by generated Triton Dockerfiles (triton-server uid)
+RUN chown 1000:1000 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \
+  && chmod 755 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so
+

Dockerfile.QA

@@ -1,4 +1,4 @@
-# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -59,23 +59,35 @@ RUN apt-get update && \
             libboost-dev \
             python3-dev \
             python3-pip \
+            python3-wheel \
             python3-setuptools \
             python3-venv \
-            python3-wheel \
             rapidjson-dev \
             software-properties-common && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install cmake==4.0.3
-ENV CMAKE_POLICY_VERSION_MINIMUM=3.5
+RUN apt update -q=2 \
+    && apt install -y gpg wget \
+    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - |  tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
+    && . /etc/os-release \
+    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
+    && apt-get update -q=2 \
+    && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3*
 
-# Add densenet_onnx model to example repo
+# Add inception_graphdef model to example repo
 # FIXME: This should be changed to using the fetch_models.sh script
 # in order to ensure the public facing docs are up-to-date.
 WORKDIR /workspace/docs/examples/model_repository
-RUN mkdir -p densenet_onnx/1 && \
-    wget -O densenet_onnx/1/model.onnx \
-            https://github.com/onnx/models/raw/main/validated/vision/classification/densenet-121/model/densenet-7.onnx
+RUN mkdir -p model_repository/inception_onnx/1 && \
+        wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \
+            https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \
+        (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \
+        python3 -m venv tf2onnx && \
+            source ./tf2onnx/bin/activate && \
+            pip3 install "numpy<2" tensorflow tf2onnx && \
+            python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 && \
+        deactivate  && \
+        mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx
 
 # Update the qa/ directory with test executables, models, etc.
 WORKDIR /workspace
@@ -105,7 +117,7 @@ RUN mkdir -p qa/common && \
     cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \
     cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \
     cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \
-    cp -r docs/examples/model_repository/densenet_onnx qa/L0_grpc/models && \
+    cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \
     mkdir qa/L0_grpc_state_cleanup/models && \
     cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \
     mkdir qa/L0_http/models && \
@@ -114,7 +126,7 @@ RUN mkdir -p qa/common && \
     cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \
     cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \
     cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \
-    cp -r docs/examples/model_repository/densenet_onnx qa/L0_http/models && \
+    cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \
     mkdir qa/L0_https/models && \
     cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \
     mkdir qa/L0_secure_grpc/models && \
@@ -139,7 +151,6 @@ RUN mkdir -p qa/common && \
     mkdir qa/L0_data_compression/models && \
     cp -r docs/examples/model_repository/simple qa/L0_data_compression/models && \
     cp bin/data_compressor_test qa/L0_data_compression/. && \
-    cp bin/tensor_size_test qa/L0_input_validation/. && \
     cp bin/metrics_api_test qa/L0_metrics/. && \
     cp bin/response_cache_test qa/L0_response_cache/. && \
     cp bin/request_cancellation_test qa/L0_request_cancellation/. && \
@@ -174,8 +185,8 @@ RUN mkdir -p qa/custom_models/custom_sequence_int32/1 && \
         qa/custom_models/custom_dyna_sequence_int32/1/.
 
 # L0_lifecycle needs No-GPU build of identity backend.
-WORKDIR  /workspace/tritonbuild/identity
-RUN rm -rf install build && mkdir build && cd build && \
+RUN cd tritonbuild/identity && \
+    rm -rf install build && mkdir build && cd build && \
     cmake -DTRITON_ENABLE_GPU=OFF \
         -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/identity/install \
         -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \
@@ -186,18 +197,15 @@ RUN rm -rf install build && mkdir build && cd build && \
     make -j16 install
 
 # L0_backend_python test require triton_shm_monitor
-ARG TRITON_BOOST_URL="https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz"
-WORKDIR /workspace/tritonbuild/python
-RUN rm -rf install build && mkdir build && cd build && \
+RUN cd tritonbuild/python && \
+    rm -rf install build && mkdir build && cd build && \
     cmake -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/python/install \
         -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \
         -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \
         -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \
-        -DTRITON_BOOST_URL:STRING=${TRITON_BOOST_URL} \
         -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \
     make -j16 triton-shm-monitor install
 
-WORKDIR /workspace/
 RUN cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \
         qa/L0_lifecycle/. && \
     cp tritonbuild/python/install/backends/python/triton_shm_monitor*.so \
@@ -317,9 +325,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN if grep -qE '^VERSION_ID="(18\.04|20\.04|22\.04|24\.04)' /etc/os-release; then \
         apt-get update && \
         apt-get install -y --no-install-recommends \
-                libpng-dev && \
-        apt-get clean && \
-        rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; \
+                libpng-dev; \
     else \
         echo "Ubuntu version must be either 18.04, 20.04, 22.04 or 24.04" && \
         exit 1; \
@@ -330,23 +336,22 @@ RUN if grep -qE '^VERSION_ID="(18\.04|20\.04|22\.04|24\.04)' /etc/os-release; th
 RUN apt-get update && apt-get install -y --no-install-recommends \
                               curl \
                               gdb \
+                              libopencv-dev \
                               libarchive-dev \
                               libopencv-core-dev \
-                              libopencv-dev \
                               libzmq3-dev \
+                              openjdk-11-jdk \
                               nginx \
                               npm \
-                              openjdk-11-jdk \
                               protobuf-compiler \
                               python3-dev \
                               python3-pip \
                               python3-protobuf \
-                              python3-setuptools \
                               python3-wheel \
+                              python3-setuptools \
                               swig \
                               valgrind && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \
+    rm -rf /var/lib/apt/lists/*
 
 # CI/QA expects "python" executable (not python3).
 RUN rm -f /usr/bin/python && \
@@ -378,17 +383,10 @@ COPY --chown=1000:1000 --from=sdk /workspace/qa/ qa/
 
 # Remove CI tests that are meant to run only on build image and
 # install the tritonserver/triton python client APIs.
-RUN rm -fr qa/L0_copyrights qa/L0_build_variants
-
-
-RUN --mount=type=secret,id=triton_ci_pip_extra_values,env=TRITON_CI_PYPI_EXTRA_VALUES \
-    if [ -n "${TRITON_CI_PYPI_EXTRA_VALUES}" ]; then \
-        find qa/pkgs/ -maxdepth 1 -type f -name \
-        "tritonclient-*any*.whl" -exec pip3 install --upgrade ${TRITON_CI_PYPI_EXTRA_VALUES} {}[all] \; ; \
-    else \
-        find qa/pkgs/ -maxdepth 1 -type f -name \
-        "tritonclient-*any*.whl" -exec pip3 install --upgrade {}[all] \; ; \
-    fi
+RUN rm -fr qa/L0_copyrights qa/L0_build_variants && \
+    find qa/pkgs/ -maxdepth 1 -type f -name \
+    "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \
+    xargs pip3 install --upgrade
 
 ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH}