Pubmatic multi inference

.gitignore

@@ -15,3 +15,4 @@ cprofile
 qa/L0_openai/openai
 tensorrtllm_models
 custom_tokenizer
+replace-artifacts/

CMakeLists.txt

@@ -26,6 +26,21 @@
 
 cmake_minimum_required(VERSION 3.18)
 
+# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5).
+# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a
+# separate CMake invocation; that step does not inherit -D variables from the top
+# level unless they are also in the environment. If configure fails inside libevent
+# with "Compatibility with CMake < 3.5 has been removed", use either:
+#   export CMAKE_POLICY_VERSION_MINIMUM=3.5   # before cmake AND cmake --build
+# or install/use CMake 3.28.x–3.31.x for this build.
+if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0")
+  message(
+    STATUS
+    "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export "
+    "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build "
+    "(see docs/customization_guide/build.md).")
+endif()
+
 project(tritonserver LANGUAGES C CXX)
 
 include(CMakeDependentOption)
@@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF)
 option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF)
 option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF)
 
+option(TRITON_ENABLE_MYSQL_ODBC
+  "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)"
+  OFF)
+
 # Need to know if TensorRT is available when building unit tests
 option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF)
 
@@ -261,6 +280,7 @@ ExternalProject_Add(triton-server
     -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3}
     -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT}
     -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE}
+    -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC}
     -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD}
     -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX}

Dockerfile

@@ -0,0 +1,104 @@
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Derivative image: start from an official Triton server image and replace only
+# the tritonserver executable and libtritonserver.so. Backends, Python wheels,
+# and the rest of the filesystem stay unchanged from the base image.
+#
+# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX):
+#   install/bin/tritonserver
+#   install/lib/libtritonserver.so
+#
+# Prepare artifacts (from repository root):
+#   mkdir -p replace-artifacts
+#   cp install/bin/tritonserver replace-artifacts/
+#   cp install/lib/libtritonserver.so replace-artifacts/
+#   Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for
+#   optional JSON field "odbcDriverName" if the default fails.
+#
+# ODBC DSN file (odbc.ini) and triton-dmconfig.json are not baked into the image;
+# bind-mount host files to /etc/odbc.ini and /etc/triton-dmconfig.json at runtime (see Run).
+#
+# Build (from repository root; match your Triton tag, e.g. r25.03):
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \
+#     -t tritonserver:25.03-custom .
+#
+# CPU-only base example:
+#   docker build \
+#     --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \
+#     -t tritonserver:25.03-custom-cpu .
+#
+# If your base image stores libtritonserver.so under lib64:
+#   --build-arg TRITON_LIB_SUBDIR=lib64
+#
+# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.:
+#   --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04
+#
+# Run (mount model repo; mount configs directly to /etc):
+#   docker run --name triton1 -d --net=host \
+#     -v "/tmp/models:/models" \
+#     -v "/etc/triton-dmconfig.json:/etc/triton-dmconfig.json:ro" \
+#     tritonserver:25.03-custom \
+#     tritonserver \
+#     --model-repository=/models \
+#     --model-control-mode explicit \
+#     --http-port=4200 --grpc-port=4201 --metrics-port=4202
+#
+# For CPU-only, drop --gpus=all and use a CPU/min base image.
+
+ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3
+
+FROM ${BASE_IMAGE}
+
+ARG TRITON_INSTALL_PREFIX=/opt/tritonserver
+ARG TRITON_LIB_SUBDIR=lib
+
+ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04
+ARG MYSQL_ODBC_DEB_ARCH=amd64
+
+# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe
+# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc.
+USER root
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unixodbc \
+    odbcinst; \
+  DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \
+  curl -fsSL -o "/tmp/${DEB}" \
+    "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \
+  apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \
+  rm -f "/tmp/${DEB}"; \
+  rm -rf /var/lib/apt/lists/*
+
+# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in
+# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under
+# /usr/lib/<triplet>/odbc/ only. unixODBC then fails with "Can't open lib
+# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/.
+RUN set -eux; \
+  mkdir -p /usr/lib/odbc; \
+  for f in \
+      /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \
+      /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \
+  do \
+    if [ -f "${f}" ]; then \
+      ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \
+    fi; \
+  done; \
+  test -f /usr/lib/odbc/libmyodbc9w.so
+
+# Paths relative to the build context (repository root when building with ".")
+COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver
+COPY replace-artifacts/libtritonserver.so \
+  ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so
+
+# Match ownership used by generated Triton Dockerfiles (triton-server uid)
+RUN chown 1000:1000 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \
+  && chmod 755 \
+      ${TRITON_INSTALL_PREFIX}/bin/tritonserver \
+      ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so

Dockerfile.pubmatic-bt-tritonserver

@@ -0,0 +1,52 @@
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Extend the image produced by build.py (`docker build -t tritonserver`) with
+# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files.
+# Build context: repository root (same as build.py final docker build).
+#
+# Prerequisites:
+#   - Image `tritonserver` must exist.
+#   - replace-artifacts/odbc.ini
+#   - replace-artifacts/triton-dmconfig.json
+#
+# Example:
+#   docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver .
+
+FROM tritonserver
+
+ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04
+ARG MYSQL_ODBC_DEB_ARCH=amd64
+
+USER root
+
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    unixodbc \
+    odbcinst; \
+  DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \
+  curl -fsSL -o "/tmp/${DEB}" \
+    "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \
+  apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \
+  rm -f "/tmp/${DEB}"; \
+  rm -rf /var/lib/apt/lists/*
+
+RUN set -eux; \
+  mkdir -p /usr/lib/odbc; \
+  for f in \
+      /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \
+      /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \
+  do \
+    if [ -f "${f}" ]; then \
+      ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \
+    fi; \
+  done; \
+  test -f /usr/lib/odbc/libmyodbc9w.so
+
+COPY replace-artifacts/odbc.ini /etc/odbc.ini
+RUN chmod 644 /etc/odbc.ini
+
+COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json
+RUN chmod 644 /etc/triton-dmconfig.json

build.py

@@ -498,6 +498,7 @@ def core_cmake_args(components, backends, cmake_dir, install_dir):
 
     cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends))
     cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends))
+    cargs.append(cmake_core_enable("TRITON_ENABLE_MYSQL_ODBC", FLAGS.enable_mysql_odbc))
 
     cargs += cmake_core_extra_args()
     cargs.append(cmake_dir)
@@ -902,6 +903,12 @@ def install_dcgm_libraries(dcgm_version, target_machine):
 
 
 def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap):
+    buildbase_odbc_layer = ""
+    if FLAGS.enable_mysql_odbc:
+        buildbase_odbc_layer = """
+RUN yum install -y unixODBC-devel
+"""
+
     df = """
 ARG TRITON_VERSION={}
 ARG TRITON_CONTAINER_VERSION={}
@@ -959,6 +966,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap):
             xz-devel \\
             zlib-devel
 """
+    df += buildbase_odbc_layer
     if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"):
         df += """
 RUN curl -k -s -L https://github.com/ccache/ccache/archive/refs/tags/v4.10.2.tar.gz -o /tmp/ccache.tar.gz \\
@@ -1051,6 +1059,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
 SHELL ["cmd", "/S", "/C"]
 """
     else:
+        mysql_odbc_line = (
+            "            unixodbc-dev \\\n" if FLAGS.enable_mysql_odbc else ""
+        )
         df += """
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
@@ -1101,7 +1112,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
             libarchive-dev \\
             libxml2-dev \\
             libnuma-dev \\
-            wget \\
+"""
+        df += mysql_odbc_line
+        df += """            wget \\
       && rm -rf /var/lib/apt/lists/*
 
 RUN pip3 install --upgrade \\
@@ -1904,6 +1917,34 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_
         docker_script.cwd(THIS_SCRIPT_DIR)
         docker_script.cmd(finalargs, check_exitcode=True)
 
+        if (
+            FLAGS.pubmatic_bt_tritonserver_image
+            and target_platform() != "windows"
+            and target_platform() != "rhel"
+        ):
+            docker_script.blankln()
+            docker_script.commentln(8)
+            docker_script.comment(
+                "Pubmatic BT image: extend tritonserver with MySQL ODBC and config files"
+            )
+            docker_script.comment(
+                "Uses Dockerfile.pubmatic-bt-tritonserver; requires replace-artifacts/"
+            )
+            docker_script.comment()
+            pubmatic_args = [
+                "docker",
+                "build",
+                "-t",
+                "pubmatic-bt-tritonserver",
+                "-f",
+                os.path.join(
+                    THIS_SCRIPT_DIR, "Dockerfile.pubmatic-bt-tritonserver"
+                ),
+                ".",
+            ]
+            docker_script.cwd(THIS_SCRIPT_DIR)
+            docker_script.cmd(pubmatic_args, check_exitcode=True)
+
         #
         # CI base image... tritonserver_cibase
         #
@@ -2616,6 +2657,20 @@ def enable_all():
         required=False,
         help="Enable ARM MALI GPU support.",
     )
+    parser.add_argument(
+        "--enable-mysql-odbc",
+        action="store_true",
+        required=False,
+        help="Build tritonserver with MySQL ODBC connection pool. For host builds install unixodbc-dev (Debian/Ubuntu) or unixODBC-devel (RHEL). Container builds add these when this flag is set.",
+    )
+    parser.add_argument(
+        "--pubmatic-bt-tritonserver-image",
+        action="store_true",
+        required=False,
+        help='After building image "tritonserver", also build "pubmatic-bt-tritonserver" '
+        "(Dockerfile.pubmatic-bt-tritonserver: ODBC driver + replace-artifacts/odbc.ini and "
+        "replace-artifacts/triton-dmconfig.json). Ubuntu/Debian-based container builds only.",
+    )
     parser.add_argument(
         "--min-compute-capability",
         type=str,

config/database_config.sample.json

@@ -0,0 +1,13 @@
+{
+        "databaseIp" : "10.0.0.1",
+        "databasePort" : 3306,
+        "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver",
+        "primaryDSNName" : "primaryDSNName",
+        "secondaryDSNName" : "secondaryDSNName",
+        "dsnUserName" : "REPLACE_WITH_USER",
+        "dsnUserPassword" : "REPLACE_WITH_SECRET",
+        "queryRetryCount": 3,
+        "dcId": "int values",
+        "minPoolConnections": 2,
+        "maxPoolConnections": 5
+}