diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yml similarity index 100% rename from .github/workflows/pre-commit.yaml rename to .github/workflows/pre-commit.yml diff --git a/.gitignore b/.gitignore index bce94b6830..e14c1671a5 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ cprofile qa/L0_openai/openai tensorrtllm_models custom_tokenizer +replace-artifacts/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d03d85665..de1e229c2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,21 @@ cmake_minimum_required(VERSION 3.18) +# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5). +# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a +# separate CMake invocation; that step does not inherit -D variables from the top +# level unless they are also in the environment. If configure fails inside libevent +# with "Compatibility with CMake < 3.5 has been removed", use either: +# export CMAKE_POLICY_VERSION_MINIMUM=3.5 # before cmake AND cmake --build +# or install/use CMake 3.28.x–3.31.x for this build. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + message( + STATUS + "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export " + "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build " + "(see docs/customization_guide/build.md).") +endif() + project(tritonserver LANGUAGES C CXX) include(CMakeDependentOption) @@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) +option(TRITON_ENABLE_MYSQL_ODBC + "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)" + OFF) + # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) @@ -261,6 +280,7 @@ ExternalProject_Add(triton-server -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} + -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..32df8c0787 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,104 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Derivative image: start from an official Triton server image and replace only +# the tritonserver executable and libtritonserver.so. Backends, Python wheels, +# and the rest of the filesystem stay unchanged from the base image. +# +# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX): +# install/bin/tritonserver +# install/lib/libtritonserver.so +# +# Prepare artifacts (from repository root): +# mkdir -p replace-artifacts +# cp install/bin/tritonserver replace-artifacts/ +# cp install/lib/libtritonserver.so replace-artifacts/ +# Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for +# optional JSON field "odbcDriverName" if the default fails. +# +# ODBC DSN file (odbc.ini) and triton-dmconfig.json are not baked into the image; +# bind-mount host files to /etc/odbc.ini and /etc/triton-dmconfig.json at runtime (see Run). +# +# Build (from repository root; match your Triton tag, e.g. r25.03): +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \ +# -t tritonserver:25.03-custom . +# +# CPU-only base example: +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \ +# -t tritonserver:25.03-custom-cpu . +# +# If your base image stores libtritonserver.so under lib64: +# --build-arg TRITON_LIB_SUBDIR=lib64 +# +# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.: +# --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04 +# +# Run (mount model repo; mount configs directly to /etc): +# docker run --name triton1 -d --net=host \ +# -v "/tmp/models:/models" \ +# -v "/etc/triton-dmconfig.json:/etc/triton-dmconfig.json:ro" \ +# tritonserver:25.03-custom \ +# tritonserver \ +# --model-repository=/models \ +# --model-control-mode explicit \ +# --http-port=4200 --grpc-port=4201 --metrics-port=4202 +# +# For CPU-only, drop --gpus=all and use a CPU/min base image. + +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 + +FROM ${BASE_IMAGE} + +ARG TRITON_INSTALL_PREFIX=/opt/tritonserver +ARG TRITON_LIB_SUBDIR=lib + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe +# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc. +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in +# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under +# /usr/lib//odbc/ only. unixODBC then fails with "Can't open lib +# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/. +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +# Paths relative to the build context (repository root when building with ".") +COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver +COPY replace-artifacts/libtritonserver.so \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + +# Match ownership used by generated Triton Dockerfiles (triton-server uid) +RUN chown 1000:1000 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ + && chmod 755 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so diff --git a/Dockerfile.pubmatic-bt-tritonserver b/Dockerfile.pubmatic-bt-tritonserver new file mode 100644 index 0000000000..340c3b27b5 --- /dev/null +++ b/Dockerfile.pubmatic-bt-tritonserver @@ -0,0 +1,52 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Extend the image produced by build.py (`docker build -t tritonserver`) with +# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files. +# Build context: repository root (same as build.py final docker build). +# +# Prerequisites: +# - Image `tritonserver` must exist. +# - replace-artifacts/odbc.ini +# - replace-artifacts/triton-dmconfig.json +# +# Example: +# docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver . + +FROM tritonserver + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json diff --git a/build.py b/build.py index 5a0f96413a..6ea96f5218 100755 --- a/build.py +++ b/build.py @@ -498,6 +498,7 @@ def core_cmake_args(components, backends, cmake_dir, install_dir): cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends)) cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends)) + cargs.append(cmake_core_enable("TRITON_ENABLE_MYSQL_ODBC", FLAGS.enable_mysql_odbc)) cargs += cmake_core_extra_args() cargs.append(cmake_dir) @@ -902,6 +903,12 @@ def install_dcgm_libraries(dcgm_version, target_machine): def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): + buildbase_odbc_layer = "" + if FLAGS.enable_mysql_odbc: + buildbase_odbc_layer = """ +RUN yum install -y unixODBC-devel +""" + df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} @@ -959,6 +966,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): xz-devel \\ zlib-devel """ + df += buildbase_odbc_layer if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"): df += """ RUN curl -k -s -L https://github.com/ccache/ccache/archive/refs/tags/v4.10.2.tar.gz -o /tmp/ccache.tar.gz \\ @@ -1051,6 +1059,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): SHELL ["cmd", "/S", "/C"] """ else: + mysql_odbc_line = ( + " unixodbc-dev \\\n" if FLAGS.enable_mysql_odbc else "" + ) df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive @@ -1101,7 +1112,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): libarchive-dev \\ libxml2-dev \\ libnuma-dev \\ - wget \\ +""" + df += mysql_odbc_line + df += """ wget \\ && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ @@ -1904,6 +1917,34 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(finalargs, check_exitcode=True) + if ( + FLAGS.pubmatic_bt_tritonserver_image + and target_platform() != "windows" + and target_platform() != "rhel" + ): + docker_script.blankln() + docker_script.commentln(8) + docker_script.comment( + "Pubmatic BT image: extend tritonserver with MySQL ODBC and config files" + ) + docker_script.comment( + "Uses Dockerfile.pubmatic-bt-tritonserver; requires replace-artifacts/" + ) + docker_script.comment() + pubmatic_args = [ + "docker", + "build", + "-t", + "pubmatic-bt-tritonserver", + "-f", + os.path.join( + THIS_SCRIPT_DIR, "Dockerfile.pubmatic-bt-tritonserver" + ), + ".", + ] + docker_script.cwd(THIS_SCRIPT_DIR) + docker_script.cmd(pubmatic_args, check_exitcode=True) + # # CI base image... tritonserver_cibase # @@ -2616,6 +2657,20 @@ def enable_all(): required=False, help="Enable ARM MALI GPU support.", ) + parser.add_argument( + "--enable-mysql-odbc", + action="store_true", + required=False, + help="Build tritonserver with MySQL ODBC connection pool. For host builds install unixodbc-dev (Debian/Ubuntu) or unixODBC-devel (RHEL). Container builds add these when this flag is set.", + ) + parser.add_argument( + "--pubmatic-bt-tritonserver-image", + action="store_true", + required=False, + help='After building image "tritonserver", also build "pubmatic-bt-tritonserver" ' + "(Dockerfile.pubmatic-bt-tritonserver: ODBC driver + replace-artifacts/odbc.ini and " + "replace-artifacts/triton-dmconfig.json). Ubuntu/Debian-based container builds only.", + ) parser.add_argument( "--min-compute-capability", type=str, diff --git a/config/database_config.sample.json b/config/database_config.sample.json new file mode 100644 index 0000000000..061a159baf --- /dev/null +++ b/config/database_config.sample.json @@ -0,0 +1,13 @@ +{ + "databaseIp" : "10.0.0.1", + "databasePort" : 3306, + "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver", + "primaryDSNName" : "primaryDSNName", + "secondaryDSNName" : "secondaryDSNName", + "dsnUserName" : "REPLACE_WITH_USER", + "dsnUserPassword" : "REPLACE_WITH_SECRET", + "queryRetryCount": 3, + "dcId": "int values", + "minPoolConnections": 2, + "maxPoolConnections": 5 +} diff --git a/docs/client_guide/openai_readme.md b/docs/client_guide/openai_readme.md deleted file mode 120000 index 05ca8a99c5..0000000000 --- a/docs/client_guide/openai_readme.md +++ /dev/null @@ -1 +0,0 @@ -../../python/openai/README.md \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8824e2ed6a..5a892f3b31 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -90,6 +90,19 @@ endif() # find_package(re2 REQUIRED) +option(TRITON_ENABLE_MYSQL_ODBC "Build tritonserver with MySQL ODBC connection pool (requires ODBC::ODBC / unixODBC)" OFF) +if(TRITON_ENABLE_MYSQL_ODBC) + if(UNIX AND NOT APPLE AND NOT WIN32) + message( + STATUS + "TRITON_ENABLE_MYSQL_ODBC: ensure unixODBC development packages are installed " + "(e.g. Debian/Ubuntu: unixodbc-dev; RHEL: unixODBC-devel) so CMake can find ODBC." + ) + endif() + find_package(ODBC REQUIRED) + message(STATUS "Using ODBC ${ODBC_VERSION}") +endif() + # # tritonserver executable # @@ -101,8 +114,10 @@ add_executable( main.cc shared_memory_manager.cc triton_signal.cc + database_config.cc classification.h common.h + database_config.h shared_memory_manager.h triton_signal.h ) @@ -155,6 +170,17 @@ else() ) endif() +if(TRITON_ENABLE_MYSQL_ODBC) + target_sources( + main + PRIVATE + mysql_odbc_connection_pool.cc + mysql_odbc_connection_pool.h + ) + target_link_libraries(main PRIVATE ODBC::ODBC) + target_compile_definitions(main PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) +endif() + set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") @@ -180,6 +206,7 @@ target_link_libraries( triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-common-logging # from repo-common + triton-common-json # from repo-common (RapidJSON) triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) @@ -337,11 +364,14 @@ if(${TRITON_ENABLE_HTTP} list(APPEND HTTP_ENDPOINT_SRCS http_server.cc + multi_infer.cc orca_http.cc ) list(APPEND HTTP_ENDPOINT_HDRS http_server.h + http_error_json.h + http_server_macros.h orca_http.h ) @@ -739,6 +769,61 @@ if (NOT WIN32) RUNTIME DESTINATION bin ) + # + # transform (JSON -> RapidJSON, TRITONSERVER_Error reporting) + # + add_library(transform STATIC transform.cc transform.h) + + # Required when transform is linked into shared objects (e.g. py-bindings -> + # http-endpoint-library -> transform); otherwise relocations fail at link time. + set_target_properties( + transform + PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) + + target_compile_features(transform PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options( + transform + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + transform + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + target_link_libraries( + transform + PRIVATE + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-common-json # RapidJSON from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + ) + + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions(transform PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) + target_link_libraries(transform PRIVATE ODBC::ODBC) + endif() + + if((${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR + ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) AND + TARGET http-endpoint-library AND TARGET transform) + target_link_libraries(http-endpoint-library PRIVATE transform) + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions( + http-endpoint-library + PRIVATE TRITON_ENABLE_MYSQL_ODBC=1 + ) + endif() + endif() + if(${TRITON_ENABLE_GPU}) # # memory_alloc example diff --git a/src/database_config.cc b/src/database_config.cc new file mode 100644 index 0000000000..cb312dec5b --- /dev/null +++ b/src/database_config.cc @@ -0,0 +1,204 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "database_config.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::optional ReadEntireFile(const std::string& path, std::string* contents) +{ + std::ifstream in(path, std::ios::binary); + if (!in) { + return std::string("failed to open file: ") + path; + } + std::ostringstream ss; + ss << in.rdbuf(); + if (!in && !in.eof()) { + return std::string("failed to read file: ") + path; + } + *contents = ss.str(); + return std::nullopt; +} + +std::optional ExpectString(const rapidjson::Value& obj, const char* key, std::string* field, bool required) +{ + if (!obj.HasMember(key)) { + if (required) { + return std::string("missing required JSON field: ") + key; + } + field->clear(); + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsString()) { + return std::string("JSON field '") + key + "' must be a string"; + } + *field = v.GetString(); + return std::nullopt; +} + +std::optional OptionalInt(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsInt()) { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = v.GetInt(); + return std::nullopt; +} + +std::optional OptionalIntFlexible(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + int n = 0; + if (v.IsInt()) { + n = v.GetInt(); + } else if (v.IsUint()) { + if (v.GetUint() > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + const int64_t v64 = v.GetInt64(); + if (v64 < INT_MIN || v64 > INT_MAX) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else if (v.IsUint64()) { + const uint64_t v64 = v.GetUint64(); + if (v64 > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = n; + return std::nullopt; +} + +std::optional OptionalNonNegativeSize( + const rapidjson::Value& obj, const char* key, std::size_t* out, + std::size_t def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (v.IsUint64()) { + *out = static_cast(v.GetUint64()); + return std::nullopt; + } + if (v.IsInt64()) { + const int64_t n = v.GetInt64(); + if (n < 0) { + return std::string("JSON field '") + key + "' must be non-negative"; + } + *out = static_cast(n); + return std::nullopt; + } + return std::string("JSON field '") + key + "' must be an integer"; +} + +} // namespace + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out) +{ + std::string raw; + if (auto e = ReadEntireFile(path, &raw)) { + return e; + } + + rapidjson::Document doc; + doc.Parse(raw.c_str()); + if (doc.HasParseError()) { + return std::string("JSON parse error: ") +rapidjson::GetParseError_En(doc.GetParseError()) + " at offset " + std::to_string(doc.GetErrorOffset()); + } + if (!doc.IsObject()) { + return std::string("root JSON value must be an object"); + } + + DatabaseConfig c; + if (auto e = ExpectString(doc, "databaseIp", &c.database_ip, false)) return e; + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.front()))) c.database_ip.erase(0, 1); + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.back()))) c.database_ip.pop_back(); + + if (auto e = OptionalIntFlexible(doc, "databasePort", &c.database_port, 3306)) return e; + + if (c.database_port < 1 || c.database_port > 65535) return std::string("databasePort must be between 1 and 65535"); + + if (auto e = ExpectString(doc, "odbcDriverName", &c.odbc_driver_name, false)) return e; + + if (auto e = ExpectString(doc, "primaryDSNName", &c.primary_dsn_name, true)) return e; + + if (auto e = ExpectString(doc, "secondaryDSNName", &c.secondary_dsn_name, false)) return e; + + if (auto e = ExpectString(doc, "dsnUserName", &c.dsn_user_name, true)) return e; + + if (auto e = ExpectString(doc, "dsnUserPassword", &c.dsn_user_password, true)) return e; + + int dc_id_int = 0; + if (auto e = OptionalIntFlexible(doc, "dcId", &dc_id_int, 0)) return e; + if (dc_id_int < INT32_MIN || dc_id_int > INT32_MAX) return std::string("dcId is out of range for int32_t"); + c.dc_id = static_cast(dc_id_int); + + if (auto e = OptionalInt(doc, "queryRetryCount", &c.query_retry_count, 3)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "minPoolConnections", &c.min_pool_connections, 2)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "maxPoolConnections", &c.max_pool_connections, 5)) return e; + + if (c.min_pool_connections < 1) return std::string("minPoolConnections must be at least 1"); + + if (c.max_pool_connections < c.min_pool_connections) return std::string("maxPoolConnections must be greater than or equal to minPoolConnections"); + *out = std::move(c); + return std::nullopt; +} + +}} // namespace triton::server + diff --git a/src/database_config.h b/src/database_config.h new file mode 100644 index 0000000000..c1083ed020 --- /dev/null +++ b/src/database_config.h @@ -0,0 +1,56 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +namespace triton { namespace server { + +inline constexpr const char kTritonDmConfigJsonPath[] = "/etc/triton-dmconfig.json"; + +struct DatabaseConfig { + std::string database_ip; + int database_port{3306}; + std::string odbc_driver_name; + + std::string primary_dsn_name; + std::string secondary_dsn_name; + std::string dsn_user_name; + std::string dsn_user_password; + + int32_t dc_id{0}; + int query_retry_count{3}; + std::size_t min_pool_connections{2}; + std::size_t max_pool_connections{5}; +}; + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out); + +}} // namespace triton::server + diff --git a/src/http_error_json.h b/src/http_error_json.h new file mode 100644 index 0000000000..4445c53560 --- /dev/null +++ b/src/http_error_json.h @@ -0,0 +1,36 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Lightweight {"error":"..."} serialization for HTTP error responses. +#pragma once + +#include +#include +#include + +#include + +#include "triton/core/tritonserver.h" + +namespace triton { namespace server { + +inline void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { + if (message == nullptr) { + message = ""; + } + + rapidjson::StringBuffer sb; + sb.Reserve(static_cast(std::strlen(message) + 16)); + rapidjson::Writer writer(sb); + writer.StartObject(); + writer.Key("error"); + writer.String(message); + writer.EndObject(); + + evbuffer_add(buffer, sb.GetString(), sb.GetSize()); +} + +inline void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { + EVBufferAddErrorJson(buffer, TRITONSERVER_ErrorMessage(err)); +} + +}} // namespace triton::server diff --git a/src/http_server.cc b/src/http_server.cc index edec3aae0a..2803885708 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -35,16 +35,24 @@ #include #include +#include +#include #include #include - +#include +#include "triton/common/triton_json.h" #include "classification.h" +#include "http_error_json.h" +#include "http_server_macros.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include +#endif #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr -#include "triton/common/triton_json.h" namespace triton { namespace server { @@ -58,36 +66,6 @@ namespace triton { namespace server { } \ } while (false) -#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ - do { \ - TRITONSERVER_Error* err__ = (X); \ - if (err__ != nullptr) { \ - EVBufferAddErrorJson((REQ)->buffer_out, err__); \ - evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ - TRITONSERVER_ErrorDelete(err__); \ - return; \ - } \ - } while (false) - -#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ - do { \ - EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ - evhtp_send_reply((REQ), CODE); \ - return; \ - } while (false) - -#define RETURN_AND_RESPOND_IF_RESTRICTED( \ - REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ - do { \ - auto const& is_restricted_api = \ - RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ - auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ - if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ - return; \ - } \ - } while (false) - - namespace { int @@ -116,26 +94,6 @@ HttpCodeFromError(TRITONSERVER_Error* error) return EVHTP_RES_BADREQ; } -void -EVBufferAddErrorJson(evbuffer* buffer, const char* message) -{ - triton::common::TritonJson::Value response( - triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - -void -EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) -{ - const char* message = TRITONSERVER_ErrorMessage(err); - EVBufferAddErrorJson(buffer, message); -} - void AddContentTypeHeader(evhtp_request_t* req, const char* type) { @@ -3166,8 +3124,13 @@ HTTPAPIServer::DecompressBuffer( case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { *decompressed_buffer = evbuffer_new(); - RETURN_IF_ERR(DataCompressor::DecompressData( - compression_type, req->buffer_in, *decompressed_buffer)); + TRITONSERVER_Error* decompress_err = DataCompressor::DecompressData( + compression_type, req->buffer_in, *decompressed_buffer); + if (decompress_err != nullptr) { + evbuffer_free(*decompressed_buffer); + *decompressed_buffer = nullptr; + return decompress_err; + } break; } case DataCompressor::Type::UNKNOWN: { @@ -3226,6 +3189,57 @@ HTTPAPIServer::ForwardHeaders( return nullptr; // success } +TRITONSERVER_Error* +HTTPAPIServer::ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse* response, const uint32_t flags, + void* userp)) +{ + RETURN_IF_ERR(ForwardHeaders(req, irequest)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, InferRequestClass::InferRequestComplete, + request_release_payload)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator_, + reinterpret_cast(&infer_request->alloc_payload_), + infer_response_complete_fn, reinterpret_cast(infer_request))); + return TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); +} + +TRITONSERVER_Error* HTTPAPIServer::FillMultiInferSlotTritonRequest(const std::string& model_name, triton::common::TritonJson::Value& infer_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req) { + RETURN_IF_ERR(ParseJsonTritonRequestID(infer_json, irequest)); + RETURN_IF_ERR(ParseJsonTritonParams(infer_json, irequest, infer_req)); + int v_idx = 0; + RETURN_IF_ERR(ParseJsonTritonIO(infer_json, irequest, infer_req, model_name, nullptr, &v_idx, 0, 0)); + return nullptr; // success +} + +#ifdef TRITON_ENABLE_MYSQL_ODBC +TRITONSERVER_Error* HTTPAPIServer::AddJsonRequestedOutput(TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, const char* output_name, uint32_t class_cnt) { + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output_name)); + infer_req->alloc_payload_.output_map_.emplace(std::piecewise_construct, std::forward_as_tuple(output_name), std::forward_as_tuple(new AllocPayload::OutputInfo(AllocPayload::OutputInfo::JSON, class_cnt))); + return nullptr; // success +} + +TRITONSERVER_Error* HTTPAPIServer::FillImpsTritonRequest(TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, ImpsInferSlot&& slot) { + const int64_t shape[] = {static_cast(slot.rows), static_cast(slot.feature_count)}; + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput(irequest, kImpsInputTensorName, TRITONSERVER_TYPE_FP32, shape, 2)); + + infer_req->serialized_data_.emplace_back(std::move(slot.input_tensor)); + std::vector& storage = infer_req->serialized_data_.back(); + const size_t byte_size = storage.size(); + + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData(irequest, kImpsInputTensorName, (byte_size > 0) ? static_cast(storage.data()) : nullptr, byte_size, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); + + RETURN_IF_ERR(AddJsonRequestedOutput(irequest, infer_req, kImpsOutputTensorName, 0)); + return nullptr; // success +} +#endif // TRITON_ENABLE_MYSQL_ODBC + void HTTPAPIServer::HandleGenerate( evhtp_request_t* req, const std::string& model_name, @@ -3694,25 +3708,11 @@ HTTPAPIServer::HandleInfer( request_id = ""; } - RETURN_AND_CALLBACK_IF_ERR(ForwardHeaders(req, irequest), error_callback); - auto request_release_payload = std::make_unique( irequest_shared, decompressed_buffer); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetReleaseCallback( - irequest, InferRequestClass::InferRequestComplete, - request_release_payload.get()), - error_callback); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator_, - reinterpret_cast(&infer_request->alloc_payload_), - InferRequestClass::InferResponseComplete, - reinterpret_cast(infer_request.get())), - error_callback); - - auto err = - TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); + auto err = ScheduleInferAsync( + req, irequest, infer_request.get(), request_release_payload.get(), + triton_trace); #ifdef TRITON_ENABLE_TRACING // Ownership of trace passed to Triton core, set trace to null to mark it // as no longer owned here. @@ -3775,17 +3775,23 @@ HTTPAPIServer::InferRequestClass::InferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request, bool register_fini_cancel_hook) : server_(server), req_(req), response_compression_type_(response_compression_type), response_count_(0), + register_fini_cancel_hook_(register_fini_cancel_hook), triton_request_(triton_request), shm_manager_(shm_manager) { evhtp_connection_t* htpconn = evhtp_request_get_connection(req); thread_ = htpconn->thread; - evhtp_request_pause(req); - evhtp_request_set_hook( - req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, - reinterpret_cast(this)); + if (pause_http_request) { + evhtp_request_pause(req); + } + if (register_fini_cancel_hook_) { + evhtp_request_set_hook( + req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, + reinterpret_cast(this)); + } } void @@ -3832,7 +3838,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( std::to_string(infer_request->response_count_)) .c_str()); } else if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( @@ -3862,7 +3868,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -3959,6 +3965,16 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( // Handle data. SHM outputs will not have an info. auto info = reinterpret_cast(userp); + if (json_only_out != nullptr) { + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "multi_infer sub-request: only plain JSON outputs are supported (no " + "shared memory, binary tensor data, or classification)"); + } + } + size_t element_count = 1; uint32_t batch_size = 0; @@ -4081,11 +4097,27 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( RETURN_IF_ERR(response_json.Add("outputs", std::move(response_outputs))); + triton::common::TritonJson::WriteBuffer json_wb; + RETURN_IF_ERR(response_json.Write(&json_wb)); + const size_t json_byte_size = json_wb.Size(); + + if (json_only_out != nullptr) { + if (!ordered_buffers.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "multi_infer sub-request: binary outputs are not supported"); + } + evbuffer_add(json_only_out, json_wb.Base(), json_byte_size); + return nullptr; // success + } + + const bool use_identity_compression = (response_compression_type_ == DataCompressor::Type::IDENTITY) || (response_compression_type_ == DataCompressor::Type::UNKNOWN); + if (ordered_buffers.empty() && use_identity_compression) { + SetResponseHeader(false, json_byte_size); + evbuffer_add(req_->buffer_out, json_wb.Base(), json_byte_size); + return nullptr; // success + } + evbuffer* response_placeholder = evbuffer_new(); - // Write json metadata into response evbuffer - triton::common::TritonJson::WriteBuffer buffer; - RETURN_IF_ERR(response_json.Write(&buffer)); - evbuffer_add(response_placeholder, buffer.Base(), buffer.Size()); + evbuffer_add(response_placeholder, json_wb.Base(), json_byte_size); // If there is binary data write it next in the appropriate // order... also need the HTTP header when returning binary data. @@ -4100,15 +4132,12 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { auto compressed_buffer = evbuffer_new(); - auto err = DataCompressor::CompressData( - response_compression_type_, response_placeholder, compressed_buffer); + auto err = DataCompressor::CompressData(response_compression_type_, response_placeholder, compressed_buffer); if (err == nullptr) { response_body = compressed_buffer; evbuffer_free(response_placeholder); } else { - // just log the compression error and return the uncompressed data - LOG_VERBOSE(1) << "unable to compress response: " - << TRITONSERVER_ErrorMessage(err); + LOG_VERBOSE(1) << "unable to compress response: " << TRITONSERVER_ErrorMessage(err); TRITONSERVER_ErrorDelete(err); evbuffer_free(compressed_buffer); response_compression_type_ = DataCompressor::Type::IDENTITY; @@ -4117,18 +4146,305 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( } case DataCompressor::Type::IDENTITY: case DataCompressor::Type::UNKNOWN: - // Do nothing for other cases break; } - SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); + SetResponseHeader(!ordered_buffers.empty(), json_byte_size); evbuffer_add_buffer(req_->buffer_out, response_body); - // Destroy the evbuffer object as the data has been moved - // to HTTP response buffer evbuffer_free(response_body); return nullptr; // success } +namespace { + +TRITONSERVER_Error* CopyTritonTensorPayloadToDoubles(const void* base, TRITONSERVER_DataType dtype, int64_t element_count, std::vector* out) { + out->resize(static_cast(element_count)); + switch (dtype) { + case TRITONSERVER_TYPE_BOOL: { + const uint8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = p[i] ? 1.0 : 0.0; + } + break; + } + case TRITONSERVER_TYPE_UINT8: { + const uint8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT16: { + const uint16_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT32: { + const uint32_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT64: { + const uint64_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT8: { + const int8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT16: { + const int16_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT32: { + const int32_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT64: { + const int64_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = p[i]; + } + break; + } + case TRITONSERVER_TYPE_FP16: + case TRITONSERVER_TYPE_BF16: + case TRITONSERVER_TYPE_BYTES: + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "tensor datatype not supported for direct multi_infer row extraction"); + case TRITONSERVER_TYPE_INVALID: + default: + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid or unsupported tensor datatype for row extraction"); + } + return nullptr; +} + +} // namespace + +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRowMajorDoubles(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out) { + rows_out->clear(); + if (expect_rows == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); + } + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); + + uint32_t output_count = 0; + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); + if (output_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "response has no outputs"); + } + + constexpr uint32_t kIdx = 0; + const char* cname = nullptr; + TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_INVALID; + const int64_t* shape = nullptr; + uint64_t dim_count = 0; + const void* base = nullptr; + size_t byte_size = 0; + TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; + int64_t memory_type_id = 0; + void* userp = nullptr; + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput(response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); + + auto* info = reinterpret_cast(userp); + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); + } + + int64_t element_count = 1; + for (uint64_t j = 0; j < dim_count; ++j) { + if (shape[j] < 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "negative dimension in output shape"); + } + element_count *= shape[j]; + } + if (element_count <= 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output has zero elements"); + } + if (element_count % static_cast(expect_rows) != 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output element count incompatible with expect_rows"); + } + + const int64_t elems_per_row = element_count / static_cast(expect_rows); + const size_t type_byte = TRITONSERVER_DataTypeByteSize(datatype); + const size_t expected_byte_size = static_cast(element_count) * type_byte; + if (expected_byte_size > byte_size) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "output byte_size too small for datatype/shape"); + } + + rows_out->resize(expect_rows); + const size_t epr = static_cast(elems_per_row); + + switch (datatype) { + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + if (epr == 1) { + for (size_t r = 0; r < expect_rows; ++r) { + (*rows_out)[r].resize(1); + (*rows_out)[r][0] = static_cast(p[r]); + } + } else { + for (size_t r = 0; r < expect_rows; ++r) { + auto& row = (*rows_out)[r]; + row.resize(epr); + const float* src = p + r * epr; + for (size_t j = 0; j < epr; ++j) { + row[j] = static_cast(src[j]); + } + } + } + return nullptr; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + if (epr == 1) { + for (size_t r = 0; r < expect_rows; ++r) { + (*rows_out)[r].resize(1); + (*rows_out)[r][0] = p[r]; + } + } else { + for (size_t r = 0; r < expect_rows; ++r) { + auto& row = (*rows_out)[r]; + row.resize(epr); + const double* src = p + r * epr; + for (size_t j = 0; j < epr; ++j) { + row[j] = src[j]; + } + } + } + return nullptr; + } + default: + break; + } + + std::vector flat; + TRITONSERVER_Error* cerr = CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); + if (cerr != nullptr) { + return cerr; + } + + for (size_t r = 0; r < expect_rows; ++r) { + const size_t off = r * epr; + (*rows_out)[r].assign(flat.begin() + static_cast(off), flat.begin() + static_cast(off + epr)); + } + return nullptr; +} + +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsScalars(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector* scores_out) { + scores_out->clear(); + if (expect_rows == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); + } + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); + + uint32_t output_count = 0; + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); + if (output_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "response has no outputs"); + } + + constexpr uint32_t kIdx = 0; + const char* cname = nullptr; + TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_INVALID; + const int64_t* shape = nullptr; + uint64_t dim_count = 0; + const void* base = nullptr; + size_t byte_size = 0; + TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; + int64_t memory_type_id = 0; + void* userp = nullptr; + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput(response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); + + auto* info = reinterpret_cast(userp); + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); + } + + int64_t element_count = 1; + for (uint64_t j = 0; j < dim_count; ++j) { + if (shape[j] < 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "negative dimension in output shape"); + } + element_count *= shape[j]; + } + if (element_count <= 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output has zero elements"); + } + if (static_cast(element_count) != expect_rows) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "scalar extract requires one output element per batch row"); + } + + const size_t type_byte = TRITONSERVER_DataTypeByteSize(datatype); + const size_t expected_byte_size = static_cast(element_count) * type_byte; + if (expected_byte_size > byte_size) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "output byte_size too small for datatype/shape"); + } + + scores_out->resize(expect_rows); + switch (datatype) { + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + std::copy(p, p + expect_rows, scores_out->begin()); + return nullptr; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + for (size_t r = 0; r < expect_rows; ++r) { + (*scores_out)[r] = static_cast(p[r]); + } + return nullptr; + } + default: + break; + } + + std::vector flat; + TRITONSERVER_Error* cerr = CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); + if (cerr != nullptr) { + return cerr; + } + for (size_t r = 0; r < expect_rows; ++r) { + (*scores_out)[r] = static_cast(flat[r]); + } + return nullptr; +} + void HTTPAPIServer::InferRequestClass::SetResponseHeader( bool has_binary_data, size_t header_length) @@ -4194,7 +4510,7 @@ HTTPAPIServer::GenerateRequestClass::InferResponseComplete( TRITONSERVER_Error* err = nullptr; if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); } if (err != nullptr) { infer_request->AddErrorJson(err); @@ -4354,8 +4670,13 @@ HTTPAPIServer::GenerateRequestClass::SendChunkResponse(bool end) TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { + if (json_only_out != nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "JSON-only response aggregation is not supported for generate"); + } triton_response_ = response; RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -4635,12 +4956,19 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingOutput( return nullptr; // success } -void -HTTPAPIServer::Handle(evhtp_request_t* req) -{ - LOG_VERBOSE(1) << "HTTP request: " << req->method << " " - << req->uri->path->full; +void HTTPAPIServer::Handle(evhtp_request_t* req) { + LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; + if (std::string(req->uri->path->full) == "/v2/multi_infer") { + HandleMultiInfer(req); + return; + } +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (std::string(req->uri->path->full) == "/v2/predict") { + HandlePredict(req); + return; + } +#endif if (std::string(req->uri->path->full) == "/v2/models/stats") { // model statistics HandleModelStats(req); diff --git a/src/http_server.h b/src/http_server.h index 785c408041..ca1ae55bdc 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -36,6 +36,7 @@ #include #include #include +#include #include "common.h" #include "data_compressor.h" @@ -48,6 +49,10 @@ namespace triton { namespace server { +#ifdef TRITON_ENABLE_MYSQL_ODBC +struct ImpsInferSlot; +#endif + class MappingSchema { public: enum class Kind { @@ -278,11 +283,13 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager); + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true); virtual ~InferRequestClass() { - if (req_ != nullptr) { + if (req_ != nullptr && register_fini_cancel_hook_) { evhtp_request_unset_hook(req_, evhtp_hook_on_request_fini); } req_ = nullptr; @@ -319,8 +326,19 @@ class HTTPAPIServer : public HTTPServer { static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); + // When json_only_out is non-null, write infer response JSON there only + // (no HTTP headers); used by POST /v2/multi_infer aggregation. virtual TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response); + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr); + + // Direct tensor read for multi_infer imps folding (skips infer JSON build). + TRITONSERVER_Error* ExtractFirstJsonOutputAsRowMajorDoubles( + TRITONSERVER_InferenceResponse* response, size_t expect_rows, + std::vector>* rows_out); + TRITONSERVER_Error* ExtractFirstJsonOutputAsScalars( + TRITONSERVER_InferenceResponse* response, size_t expect_rows, + std::vector* scores_out); // Helper function to set infer response header in the form specified by // the endpoint protocol @@ -329,6 +347,8 @@ class HTTPAPIServer : public HTTPServer { uint32_t IncrementResponseCount(); + uint32_t ResponseCount() const { return response_count_.load(); } + // Only used if tracing enabled std::shared_ptr trace_; @@ -359,6 +379,8 @@ class HTTPAPIServer : public HTTPServer { // Counter to keep track of number of responses generated. std::atomic response_count_{0}; + const bool register_fini_cancel_hook_; + // Event hook for called before request deletion static evhtp_res RequestFiniHook(evhtp_request* req, void* arg); @@ -412,7 +434,8 @@ class HTTPAPIServer : public HTTPServer { // Response preparation TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response) override; + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr) override; void AddErrorJson(TRITONSERVER_Error* error); static void StartResponse(evthr_t* thr, void* arg, void* shared); @@ -506,11 +529,13 @@ class HTTPAPIServer : public HTTPServer { // [FIXME] extract to "infer" class virtual std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) { return std::unique_ptr(new InferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } // Helper function to retrieve infer request header in the form specified by @@ -541,6 +566,17 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Error* ForwardHeaders( evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest); + // ForwardHeaders, release/response callbacks, and ServerInferAsync (shared + // by HandleInfer and multi_infer sub-requests). + TRITONSERVER_Error* ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse*, const uint32_t, void*) = + InferRequestClass::InferResponseComplete); + static TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, @@ -573,6 +609,12 @@ class HTTPAPIServer : public HTTPServer { void HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); +#ifdef TRITON_ENABLE_MYSQL_ODBC + // POST /v2/predict — imps-shaped BT inference (feature mapping + model routing). + void HandlePredict(evhtp_request_t* req); +#endif // TRITON_ENABLE_MYSQL_ODBC + // POST /v2/multi_infer — parallel infer for multiple models (requests array). + void HandleMultiInfer(evhtp_request_t* req); void HandleModelStats( evhtp_request_t* req, const std::string& model_name = "", const std::string& model_version_str = ""); @@ -636,6 +678,23 @@ class HTTPAPIServer : public HTTPServer { triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest); + // Fills irequest from a multi_infer slot object (inputs/outputs/id/parameters). + // JSON tensor data only; no trailing binary block (v/n/header_length unused). + TRITONSERVER_Error* FillMultiInferSlotTritonRequest( + const std::string& model_name, + triton::common::TritonJson::Value& infer_json, + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req); + +#ifdef TRITON_ENABLE_MYSQL_ODBC + static TRITONSERVER_Error* AddJsonRequestedOutput( + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, + const char* output_name, uint32_t class_cnt = 0); + + static TRITONSERVER_Error* FillImpsTritonRequest( + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, + ImpsInferSlot&& slot); +#endif // TRITON_ENABLE_MYSQL_ODBC + std::shared_ptr server_; // Storing server metadata as it is consistent during server running diff --git a/src/http_server_macros.h b/src/http_server_macros.h new file mode 100644 index 0000000000..df6f38c900 --- /dev/null +++ b/src/http_server_macros.h @@ -0,0 +1,42 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Shared preprocessor macros for HTTP API handlers. Macros are not C++ symbols +// and are not "in" a namespace; this header is included from within +// triton::server for consistency with http_server.cc. +// +// Prerequisites at expansion sites: HttpCodeFromError, EVBufferAddErrorJson +// (from http_error_json.h), +// and (for RETURN_AND_RESPOND_IF_RESTRICTED) RespondIfRestricted must be +// visible — typically from the same translation unit's anonymous namespace and +// HTTPAPIServer member functions respectively. + +#pragma once + +#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ + do { \ + TRITONSERVER_Error* err__ = (X); \ + if (err__ != nullptr) { \ + EVBufferAddErrorJson((REQ)->buffer_out, err__); \ + evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ + TRITONSERVER_ErrorDelete(err__); \ + return; \ + } \ + } while (false) + +#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ + do { \ + EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ + evhtp_send_reply((REQ), CODE); \ + return; \ + } while (false) + +#define RETURN_AND_RESPOND_IF_RESTRICTED( \ + REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ + do { \ + auto const& is_restricted_api = \ + RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ + auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ + if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ + return; \ + } \ + } while (false) diff --git a/src/main.cc b/src/main.cc index b2eee17c68..c09f1a3bba 100644 --- a/src/main.cc +++ b/src/main.cc @@ -42,11 +42,21 @@ #include #include +#include +#include +#include #include #include +#include +#include #include #include +#include "database_config.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "mysql_odbc_connection_pool.h" +#include "transform.h" +#endif // TRITON_ENABLE_MYSQL_ODBC #include "triton_signal.h" #ifdef TRITON_ENABLE_ASAN @@ -103,6 +113,113 @@ std::unique_ptr g_vertex_ai_service; triton::server::TritonServerParameters g_triton_params; +// Populated at startup when /etc/triton-dmconfig.json is present. +std::optional g_triton_dm_database_config; + +#ifdef TRITON_ENABLE_MYSQL_ODBC +// ODBC pool opened after successful config load (same lifetime as the process). +std::unique_ptr g_triton_dm_odbc_pool; + +// Registered with std::atexit after the pool is initialized so connections are +// released on normal return, exit(), and FAIL_IF_ERR paths. +extern "C" void TritonDmOdbcPoolAtExit(void) +{ + triton::server::SetGlobalMysqlOdbcPool(nullptr); + g_triton_dm_odbc_pool.reset(); +} + +void TritonModelsRefreshThreadMain() +{ + using std::chrono_literals::operator""min; + while (!triton::server::signal_exiting_) { + if (auto err = triton::server::UpdateTritonModelsData()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Triton ML models DB refresh failed: " << *err; +#endif // TRITON_ENABLE_LOGGING + } + std::unique_lock lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.wait_for(lock, 15min, [] { return triton::server::signal_exiting_; }); + } +} + +std::optional g_triton_models_refresh_thread; + +void StartTritonModelsRefreshThread() +{ + if (!g_triton_dm_odbc_pool) { + return; + } + g_triton_models_refresh_thread.emplace(TritonModelsRefreshThreadMain); +} + +void JoinTritonModelsRefreshThread() +{ + if (!g_triton_models_refresh_thread.has_value()) { + return; + } + if (g_triton_models_refresh_thread->joinable()) { + { + std::lock_guard lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.notify_all(); + } + g_triton_models_refresh_thread->join(); + } + g_triton_models_refresh_thread.reset(); +} +#endif // TRITON_ENABLE_MYSQL_ODBC + +void LoadTritonDmDatabaseConfigAtStartup() +{ + namespace fs = std::filesystem; + const std::string path(triton::server::kTritonDmConfigJsonPath); + if (!fs::exists(path)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Database config file '" << path << "' not found; continuing without DM database metadata."; +#else + std::cerr << "warning: database config file '" << path << "' not found; continuing without DM database metadata." << std::endl; +#endif // TRITON_ENABLE_LOGGING + return; + } + + triton::server::DatabaseConfig cfg; + if (auto err = triton::server::LoadDatabaseConfigFromJsonFile(path, &cfg)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to load '" << path << "': " << *err; +#else + std::cerr << "Failed to load '" << path << "': " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + exit(1); + } + + g_triton_dm_database_config = std::move(cfg); + +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "Loaded database config from '" << path << "' (databaseIp='" << g_triton_dm_database_config->database_ip << "', databasePort=" << g_triton_dm_database_config->database_port << ")"; +#endif // TRITON_ENABLE_LOGGING + +#ifdef TRITON_ENABLE_MYSQL_ODBC + g_triton_dm_odbc_pool = std::make_unique(*g_triton_dm_database_config); + if (auto err = g_triton_dm_odbc_pool->Initialize()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to initialize MySQL ODBC connection pool: " << *err; +#else + std::cerr << "Failed to initialize MySQL ODBC connection pool: " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + g_triton_dm_odbc_pool.reset(); + exit(1); + } + if (std::atexit(TritonDmOdbcPoolAtExit) != 0) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "std::atexit failed; ODBC pool will still be destroyed at static exit"; +#endif // TRITON_ENABLE_LOGGING + } + triton::server::SetGlobalMysqlOdbcPool(g_triton_dm_odbc_pool.get()); +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "MySQL ODBC connection pool started ("<< g_triton_dm_database_config->max_pool_connections << " connections to DSN '" << g_triton_dm_database_config->primary_dsn_name << "')"; +#endif // TRITON_ENABLE_LOGGING +#endif // TRITON_ENABLE_MYSQL_ODBC +} + #ifdef TRITON_ENABLE_GRPC TRITONSERVER_Error* StartGrpcService( @@ -469,6 +586,8 @@ main(int argc, char** argv) LOG_SET_OUT_FILE(g_triton_params.log_file_); #endif // TRITON_ENABLE_LOGGING + LoadTritonDmDatabaseConfigAtStartup(); + // Trace manager. triton::server::TraceManager* trace_manager; @@ -501,6 +620,11 @@ main(int argc, char** argv) exit(1); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + StartTritonModelsRefreshThread(); + FAIL_IF_ERR(triton::server::InitializeReadyModelNames(server_ptr), "initializing ready model names"); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Wait until a signal terminates the server... while (!triton::server::signal_exiting_) { // If enabled, poll the model repository to see if there have been @@ -521,6 +645,10 @@ main(int argc, char** argv) triton::server::signal_exit_cv_.wait_for(lock, wait_timeout); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + JoinTritonModelsRefreshThread(); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Stop the HTTP[, gRPC, and metrics] endpoints, and update exit timeout. uint32_t exit_timeout_secs = g_triton_params.exit_timeout_secs_; StopEndpoints(&exit_timeout_secs); diff --git a/src/multi_infer.cc b/src/multi_infer.cc new file mode 100644 index 0000000000..588d65966c --- /dev/null +++ b/src/multi_infer.cc @@ -0,0 +1,979 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "http_server.h" + +#include "common.h" +#include "http_error_json.h" +#include "http_server_macros.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +int HttpCodeFromError(TRITONSERVER_Error* error) { + if (error == nullptr) { + return EVHTP_RES_OK; + } + switch (TRITONSERVER_ErrorCode(error)) { + case TRITONSERVER_ERROR_INTERNAL: + return EVHTP_RES_SERVERR; + case TRITONSERVER_ERROR_NOT_FOUND: + return EVHTP_RES_NOTFOUND; + case TRITONSERVER_ERROR_UNAVAILABLE: + return EVHTP_RES_SERVUNAVAIL; + case TRITONSERVER_ERROR_UNSUPPORTED: + return EVHTP_RES_NOTIMPL; + case TRITONSERVER_ERROR_UNKNOWN: + case TRITONSERVER_ERROR_INVALID_ARG: + case TRITONSERVER_ERROR_ALREADY_EXISTS: + case TRITONSERVER_ERROR_CANCELLED: + return EVHTP_RES_BADREQ; + } + + return EVHTP_RES_BADREQ; +} + +void AddContentTypeHeader(evhtp_request_t* req, const char* type) { + auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); + if (content_header) { + evhtp_header_rm_and_free(req->headers_out, content_header); + } + + evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); +} + +void AppendJsonEscaped(std::string* out, const std::string& value) +{ + out->reserve(out->size() + value.size() + 8); + for (char c : value) { + switch (c) { + case '"': + out->append("\\\""); + break; + case '\\': + out->append("\\\\"); + break; + case '\b': + out->append("\\b"); + break; + case '\f': + out->append("\\f"); + break; + case '\n': + out->append("\\n"); + break; + case '\r': + out->append("\\r"); + break; + case '\t': + out->append("\\t"); + break; + default: + if (static_cast(c) < 0x20) { + char buf[7]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(c)); + out->append(buf); + } else { + out->push_back(c); + } + break; + } + } +} + +TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { + *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); + { + triton::common::TritonJson::Value v; + if (slot.Find("id", &v)) { + RETURN_IF_ERR(infer_json->Add("id", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (!slot.Find("inputs", &v)) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); + } + RETURN_IF_ERR(infer_json->Add("inputs", std::move(v))); + } + { + triton::common::TritonJson::Value v; + if (slot.Find("outputs", &v)) { + RETURN_IF_ERR(infer_json->Add("outputs", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (slot.Find("parameters", &v)) { + RETURN_IF_ERR(infer_json->Add("parameters", std::move(v))); + } + } + return nullptr; +} + +TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) +{ + ver_out->clear(); + triton::common::TritonJson::Value mv; + if (!slot.Find("model_version", &mv)) { + return nullptr; + } + if (mv.IsString()) { + const char* s; + size_t len; + RETURN_IF_ERR(mv.AsString(&s, &len)); + ver_out->assign(s, len); + return nullptr; + } + if (mv.IsNumber()) { + int64_t iv; + RETURN_IF_ERR(mv.AsInt(&iv)); + *ver_out = std::to_string(iv); + return nullptr; + } + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); +} + +#ifdef TRITON_ENABLE_MYSQL_ODBC + +inline float RoundScore6(float x) { + if (!std::isfinite(x)) { + return x; + } + const int64_t scaled = static_cast(std::floor(x * 1e6f)); + return static_cast(scaled) / 1e6f; +} + +bool WriteImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector& slot_model_names, const std::vector>& slot_rows, const int imp_count, rapidjson::StringBuffer* sb) { + if (imp_count <= 0 || routing_slots.empty() || sb == nullptr) { + return false; + } + if (slot_rows.size() != routing_slots.size() || slot_model_names.size() != routing_slots.size()) { + return false; + } + + struct CampAgg { + int32_t cid{0}; + const std::string* mdl{nullptr}; + std::vector> by_adsize; + bool initialized{false}; + }; + + size_t total_rows = 0; + for (size_t si = 0; si < routing_slots.size(); ++si) { + total_rows += routing_slots[si].size(); + } + + std::vector> camps_per_imp(static_cast(imp_count)); + for (size_t si = 0; si < routing_slots.size(); ++si) { + const auto& slot_r = routing_slots[si]; + const std::string& slot_mdl = slot_model_names[si]; + const size_t R = slot_r.size(); + if (si >= slot_rows.size()) { + return false; + } + const auto& row_scores = slot_rows[si]; + if (row_scores.size() != R) { + return false; + } + for (size_t ri = 0; ri < R; ++ri) { + const ImpRouteRow& rc = slot_r[ri]; + if (rc.imp_idx < 0 || rc.imp_idx >= imp_count || rc.camp_idx < 0) { + return false; + } + auto& imp_camps = camps_per_imp[static_cast(rc.imp_idx)]; + if (static_cast(rc.camp_idx) >= imp_camps.size()) { + imp_camps.resize(static_cast(rc.camp_idx) + 1); + } + CampAgg& ca = imp_camps[static_cast(rc.camp_idx)]; + if (!ca.initialized) { + ca.cid = rc.cid; + ca.mdl = &slot_mdl; + ca.initialized = true; + } else if (ca.cid != rc.cid || *ca.mdl != slot_mdl) { + return false; + } + const size_t ad_idx = static_cast(rc.adsize_idx); + if (ad_idx >= ca.by_adsize.size()) { + ca.by_adsize.resize(ad_idx + 1); + } + ca.by_adsize[ad_idx] = {row_scores[ri]}; + } + } + + sb->Clear(); + sb->Reserve(static_cast(128 + total_rows * 96)); + rapidjson::Writer writer(*sb); + writer.SetMaxDecimalPlaces(6); + + writer.StartObject(); + writer.Key("imps"); + writer.StartArray(); + for (int ii = 0; ii < imp_count; ++ii) { + writer.StartObject(); + writer.Key("camps"); + writer.StartArray(); + for (const CampAgg& ca : camps_per_imp[static_cast(ii)]) { + if (!ca.initialized) { + continue; + } + writer.StartObject(); + writer.Key("cid"); + writer.Int(ca.cid); + writer.Key("mdl"); + writer.String(ca.mdl->c_str(), static_cast(ca.mdl->size())); + writer.Key("score"); + writer.StartArray(); + for (const std::vector& vec : ca.by_adsize) { + if (vec.empty()) { + continue; + } + if (vec.size() == 1) { + writer.Double(static_cast(RoundScore6(vec[0]))); + } else { + writer.StartArray(); + for (float f : vec) { + writer.Double(static_cast(RoundScore6(f))); + } + writer.EndArray(); + } + } + writer.EndArray(); + writer.EndObject(); + } + writer.EndArray(); + writer.EndObject(); + } + writer.EndArray(); + writer.EndObject(); + return true; +} + +#endif // TRITON_ENABLE_MYSQL_ODBC + +class MultiInferAggregator : public std::enable_shared_from_this { + private: + struct FinishPayload { + std::shared_ptr agg; + }; + + public: + MultiInferAggregator(evhtp_request_t* req, size_t slot_count, evthr_t* reply_thread, + std::vector> irequests +#ifdef TRITON_ENABLE_MYSQL_ODBC + , std::vector> imp_routing_slots = {}, + std::vector slot_model_names = {}, + int imp_routing_imp_count = 0 +#endif + ) + : req_(req), n_(slot_count), reply_thread_(reply_thread), + irequests_(std::move(irequests)), success_buffers_(slot_count, nullptr), + error_text_(slot_count), have_error_(slot_count, 0) +#ifdef TRITON_ENABLE_MYSQL_ODBC + , imp_routing_slots_(std::move(imp_routing_slots)), + slot_model_names_(std::move(slot_model_names)), + imp_routing_imp_count_(imp_routing_imp_count) +#endif + { +#ifdef TRITON_ENABLE_MYSQL_ODBC + slot_row_outputs_.assign(n_, {}); +#endif + } + + ~MultiInferAggregator() + { + for (evbuffer* buf : success_buffers_) { + if (buf != nullptr) { + evbuffer_free(buf); + } + } + } + + std::shared_ptr IrequestAt(size_t i) const + { + return irequests_[i]; + } + +#ifdef TRITON_ENABLE_MYSQL_ODBC + bool WantsShardParsedRows() const + { + return imp_routing_imp_count_ > 0 && !imp_routing_slots_.empty(); + } + + size_t ExpectedRowsForSlot(size_t slot) const + { + return (slot < imp_routing_slots_.size()) ? imp_routing_slots_[slot].size() : 0; + } +#endif + + void CancelAllSubRequests() + { + if (cancel_sent_.exchange(true, std::memory_order_acq_rel)) { + return; + } + for (auto& ir : irequests_) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); + } + } + } + + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, evbuffer* response_json, std::vector parsed_scalar_output = {}) { + if (finalize_err != nullptr) { + have_error_[slot] = 1; + error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); + TRITONSERVER_ErrorDelete(finalize_err); + if (response_json != nullptr) { + evbuffer_free(response_json); + } + CancelAllSubRequests(); + } else { + success_buffers_[slot] = response_json; +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (!parsed_scalar_output.empty() && slot < slot_row_outputs_.size()) { + slot_row_outputs_[slot] = std::move(parsed_scalar_output); + } +#endif + } + + std::atomic_thread_fence(std::memory_order_release); + + const size_t prev = done_count_.fetch_add(1, std::memory_order_acq_rel); + if (prev + 1 < n_) { + return; + } + + bool expected = false; + if (!reply_scheduled_.compare_exchange_strong(expected, true, std::memory_order_acq_rel, std::memory_order_relaxed)) { + return; + } + + auto* fp = new FinishPayload{shared_from_this()}; + evthr_defer(reply_thread_, FinishThunk, fp); + } + + private: + static void FinishThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + std::unique_ptr fp(static_cast(arg)); + fp->agg->WriteHttpReply(); + } + + static void AppendShardErrorJson(evbuffer* out, const std::string& message) + { + std::string fragment; + fragment.reserve(message.size() + 24); + fragment += "{\"error\":{\"message\":\""; + AppendJsonEscaped(&fragment, message); + fragment += "\"}}"; + evbuffer_add(out, fragment.data(), fragment.size()); + } + + void WriteHttpReply() { + std::atomic_thread_fence(std::memory_order_acquire); + +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (WantsShardParsedRows() && !cancel_sent_.load(std::memory_order_acquire)) { + bool any_err = false; + for (size_t i = 0; i < n_; ++i) { + if (have_error_[i]) { + any_err = true; + break; + } + } + if (!any_err) { + rapidjson::StringBuffer sb; + if (WriteImpsShapedMultiInferResponse(imp_routing_slots_, slot_model_names_, slot_row_outputs_, imp_routing_imp_count_, &sb)) { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, sb.GetString(), sb.GetSize()); + evhtp_send_reply(req_, EVHTP_RES_OK); + evhtp_request_resume(req_); + return; + } + static const char kFoldErr[] = "{\"error\":\"failed to fold imps response\"}"; + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, kFoldErr, sizeof(kFoldErr) - 1); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + evhtp_request_resume(req_); + return; + } + + bool any_shard_error = false; + for (size_t i = 0; i < n_; ++i) { + if (have_error_[i]) { + any_shard_error = true; + break; + } + } + if (any_shard_error) { + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value errors(root, triton::common::TritonJson::ValueType::ARRAY); + for (size_t i = 0; i < n_; ++i) { + TRITONSERVER_Error* ae = nullptr; + if (have_error_[i]) { + ae = errors.AppendString(error_text_[i]); + } else { + ae = errors.AppendString(""); + } + if (ae != nullptr) { + LOG_TRITONSERVER_ERROR(ae, "multi_infer: building errors array"); + TRITONSERVER_ErrorDelete(ae); + } + } + TRITONSERVER_Error* re = root.Add("errors", std::move(errors)); + if (re != nullptr) { + LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, re); + evhtp_send_reply(req_, HttpCodeFromError(re)); + TRITONSERVER_ErrorDelete(re); + evhtp_request_resume(req_); + return; + } + triton::common::TritonJson::WriteBuffer wb; + TRITONSERVER_Error* we = root.Write(&wb); + if (we != nullptr) { + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, we); + evhtp_send_reply(req_, HttpCodeFromError(we)); + TRITONSERVER_ErrorDelete(we); + } else { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + } + evhtp_request_resume(req_); + return; + } + } +#endif // TRITON_ENABLE_MYSQL_ODBC + + evbuffer* out = evbuffer_new(); + evbuffer_add(out, "{\"responses\":[", 14); + + for (size_t i = 0; i < n_; ++i) { + if (i > 0) { + evbuffer_add(out, ",", 1); + } + if (have_error_[i]) { + AppendShardErrorJson(out, error_text_[i]); + } else if (success_buffers_[i] == nullptr || evbuffer_get_length(success_buffers_[i]) == 0) { + AppendShardErrorJson(out, "empty multi_infer sub-response"); + if (success_buffers_[i] != nullptr) { + evbuffer_free(success_buffers_[i]); + success_buffers_[i] = nullptr; + } + } else { + evbuffer_add_buffer(out, success_buffers_[i]); + evbuffer_free(success_buffers_[i]); + success_buffers_[i] = nullptr; + } + } + + evbuffer_add(out, "]}", 2); + + AddContentTypeHeader(req_, "application/json"); + evbuffer_add_buffer(req_->buffer_out, out); + evbuffer_free(out); + evhtp_send_reply(req_, EVHTP_RES_OK); + evhtp_request_resume(req_); + } + + evhtp_request_t* req_; + const size_t n_; + evthr_t* reply_thread_; + std::vector> irequests_; + + std::atomic done_count_{0}; + std::vector success_buffers_; + std::vector error_text_; + std::vector have_error_; + std::atomic cancel_sent_{false}; + std::atomic reply_scheduled_{false}; +#ifdef TRITON_ENABLE_MYSQL_ODBC + std::vector> imp_routing_slots_; + std::vector slot_model_names_; + std::vector> slot_row_outputs_; + int imp_routing_imp_count_{0}; +#endif +}; + +class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { + public: + MultiInferShardRequest(TRITONSERVER_Server* server, evhtp_request_t* req, + DataCompressor::Type response_compression_type, + const std::shared_ptr& triton_request, + const std::shared_ptr& shm_manager, + std::shared_ptr aggregator, const size_t slot) + : HTTPAPIServer::InferRequestClass(server, req, response_compression_type, triton_request, shm_manager, false /* pause */, false /* fini hook */), aggregator_(std::move(aggregator)), slot_(slot){} + + static void InferResponseComplete(TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { + auto* infer_request = reinterpret_cast(userp); + + if (response != nullptr) { + ++infer_request->response_count_; + } + + TRITONSERVER_Error* err = nullptr; + evbuffer* shard_json = nullptr; + std::vector pre_parsed_scalars; + if (infer_request->response_count_ != 1) { + const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); + } else if (response != nullptr) { + bool skip_shard_json = false; +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (infer_request->aggregator_->WantsShardParsedRows()) { + const size_t nrows = infer_request->aggregator_->ExpectedRowsForSlot(infer_request->slot_); + if (nrows > 0u) { + TRITONSERVER_Error* ex_err = infer_request->ExtractFirstJsonOutputAsScalars(response, nrows, &pre_parsed_scalars); + if (ex_err == nullptr) { + skip_shard_json = true; + } else { + err = ex_err; + } + } + } +#endif + if (!skip_shard_json) { + shard_json = evbuffer_new(); + err = infer_request->FinalizeResponse(response, shard_json); + } +#ifdef TRITON_ENABLE_TRACING + if (infer_request->trace_ != nullptr) { + infer_request->trace_->CaptureTimestamp("INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); + } +#endif // TRITON_ENABLE_TRACING + } + + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); + + if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { + if (shard_json != nullptr) { + evbuffer_free(shard_json); + } + return; + } + + if (err != nullptr) { + if (shard_json != nullptr) { + evbuffer_free(shard_json); + } + infer_request->aggregator_->OnShardDone(infer_request->slot_, err, nullptr); + } else { + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, shard_json, std::move(pre_parsed_scalars)); + } + evthr_defer(infer_request->thread_, DeleteMultiInferShardRequestThunk, infer_request); + } + + private: + static void DeleteMultiInferShardRequestThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + delete reinterpret_cast(arg); + } + + std::shared_ptr aggregator_; + const size_t slot_; +}; + +struct PostBodyContent { + evbuffer* decompressed_buffer{nullptr}; + TRITONSERVER_Error* read_error{nullptr}; + const char* data{""}; + size_t size{0}; +}; + +PostBodyContent ReadPostBody(evhtp_request_t* req, evbuffer* decompressed_buffer) +{ + PostBodyContent out; + out.decompressed_buffer = decompressed_buffer; + evbuffer* body_buf = (out.decompressed_buffer != nullptr) ? out.decompressed_buffer : req->buffer_in; + out.size = evbuffer_get_length(body_buf); + if (out.size > 0) { + const unsigned char* pulled = evbuffer_pullup(body_buf, -1); + if (pulled == nullptr) { + out.read_error = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read request body"); + } else { + out.data = reinterpret_cast(pulled); + } + } + return out; +} + +void FreePostBody(PostBodyContent* body) +{ + if (body->decompressed_buffer != nullptr) { + evbuffer_free(body->decompressed_buffer); + body->decompressed_buffer = nullptr; + } +} + +void RespondWithTritonError(evhtp_request_t* req, TRITONSERVER_Error* err) +{ + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); +} + +} // namespace + +#ifdef TRITON_ENABLE_MYSQL_ODBC +void HTTPAPIServer::HandlePredict(evhtp_request_t* req) { + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + + if (req->method != htp_method_POST) { + RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); + } + + evhtp_request_pause(req); + + evbuffer* decompressed_buffer = nullptr; + TRITONSERVER_Error* read_error = DecompressBuffer(req, &decompressed_buffer); + if (read_error != nullptr) { + RespondWithTritonError(req, read_error); + return; + } + + PostBodyContent body = ReadPostBody(req, decompressed_buffer); + if (body.read_error != nullptr) { + RespondWithTritonError(req, body.read_error); + FreePostBody(&body); + return; + } + + if (body.size == 0) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array")); + FreePostBody(&body); + return; + } + + rapidjson::Document imps_doc; + imps_doc.Parse(body.data, body.size); + FreePostBody(&body); + + if (imps_doc.HasParseError() || !imps_doc.IsObject() || !imps_doc.HasMember("imps") || !imps_doc["imps"].IsArray()) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array")); + return; + } + + ImpRoutingTable imp_routing; + std::vector imps_slots; + TRITONSERVER_Error* err = GenerateImpsInferSlots(imps_doc, server_.get(), &imps_slots, &imp_routing); + if (err != nullptr) { + RespondWithTritonError(req, err); + return; + } + + const size_t n = imps_slots.size(); + if (n == 0) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps request produced no inference sub-requests")); + return; + } + + std::unordered_set policy_checked_models; + for (size_t i = 0; i < n; ++i) { + if (!policy_checked_models.insert(imps_slots[i].model_name).second) { + continue; + } + err = CheckTransactionPolicy(req, imps_slots[i].model_name, imps_slots[i].model_version); + if (err != nullptr) { + RespondWithTritonError(req, err); + return; + } + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), imps_slots[i].model_name.c_str(), imps_slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused predict sub-request"); + } + } + RespondWithTritonError(req, err); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r), "deleting HTTP predict sub-request"); + }); + } + + std::vector slot_model_names; + slot_model_names.reserve(n); + for (const auto& slot : imps_slots) { + slot_model_names.push_back(slot.original_model_name.empty() ? slot.model_name : slot.original_model_name); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing.slots), std::move(slot_model_names), imp_routing.imp_count); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); + + for (size_t i = 0; i < n; ++i) { + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); + + err = FillImpsTritonRequest(irequests[i].get(), shard.get(), std::move(imps_slots[i])); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + RespondWithTritonError(req, err); + return; + } + + auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + RespondWithTritonError(req, err); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } +} +#endif // TRITON_ENABLE_MYSQL_ODBC + +void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + + if (req->method != htp_method_POST) { + RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); + } + + evhtp_request_pause(req); + + evbuffer* decompressed_buffer = nullptr; + TRITONSERVER_Error* read_error = DecompressBuffer(req, &decompressed_buffer); + if (read_error != nullptr) { + RespondWithTritonError(req, read_error); + return; + } + + PostBodyContent body = ReadPostBody(req, decompressed_buffer); + if (body.read_error != nullptr) { + RespondWithTritonError(req, body.read_error); + FreePostBody(&body); + return; + } + + triton::common::TritonJson::Value root; + TRITONSERVER_Error* err = root.Parse(body.data, body.size); + FreePostBody(&body); + if (err != nullptr) { + RespondWithTritonError(req, err); + return; + } + + triton::common::TritonJson::Value requests; + if (!root.Find("requests", &requests)) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + const size_t n = requests.ArraySize(); + if (n == 0) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + struct SlotPrep { + std::string model_name; + int64_t model_version{0}; + triton::common::TritonJson::Value infer_json; + }; + std::vector slots; + slots.reserve(n); + + for (size_t i = 0; i < n; ++i) { + triton::common::TritonJson::Value slot; + err = requests.At(i, &slot); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + const char* mn_c; + size_t mn_len; + err = slot.MemberAsString("model_name", &mn_c, &mn_len); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + SlotPrep prep; + prep.model_name.assign(mn_c, mn_len); + std::string ver_str; + err = GetModelVersionStringFromSlot(slot, &ver_str); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = GetModelVersionFromString(ver_str, &prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = CheckTransactionPolicy(req, prep.model_name, prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + triton::common::TritonJson::Value infer_only; + err = CopyInferSlotBodyJson(slot, &infer_only); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + prep.infer_json = std::move(infer_only); + slots.push_back(std::move(prep)); + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), slots[i].model_name.c_str(), slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused multi_infer sub-request"); + } + } + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r),"deleting HTTP multi_infer sub-request"); + }); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); + + for (size_t i = 0; i < n; ++i) { + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); + + err = FillMultiInferSlotTritonRequest(slots[i].model_name, slots[i].infer_json, irequests[i].get(), shard.get()); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } +} + +}} // namespace triton::server diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc new file mode 100644 index 0000000000..5ed9210293 --- /dev/null +++ b/src/mysql_odbc_connection_pool.cc @@ -0,0 +1,750 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "mysql_odbc_connection_pool.h" + +#ifdef TRITON_ENABLE_LOGGING +#include "triton/common/logging.h" +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::atomic g_global_mysql_odbc_pool{nullptr}; + +constexpr auto kPoolAcquireTimeout = std::chrono::seconds(30); + +std::string BuildMySqlDriverConnectString(const DatabaseConfig& c) +{ + std::string driver = c.odbc_driver_name; + if (driver.empty()) { + driver = "MySQL ODBC 9.7 Unicode Driver"; + } + std::ostringstream conn; + conn << "DRIVER={" << driver << "};" << "SERVER=" << c.database_ip << ";" << "PORT=" << c.database_port << ";" << "UID={" << c.dsn_user_name << "};" << "PWD={" << c.dsn_user_password << "};"; + return conn.str(); +} + +} // namespace + +PooledOdbcConnection::PooledOdbcConnection() = default; +PooledOdbcConnection::PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc) : pool_(pool), dbc_(dbc){} +PooledOdbcConnection::~PooledOdbcConnection() +{ + Release(); +} + +PooledOdbcConnection::PooledOdbcConnection(PooledOdbcConnection&& other) noexcept : pool_(other.pool_), dbc_(other.dbc_) +{ + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; +} + +PooledOdbcConnection& PooledOdbcConnection::operator=(PooledOdbcConnection&& other) noexcept +{ + if (this != &other) { + Release(); + pool_ = other.pool_; + dbc_ = other.dbc_; + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; + } + return *this; +} + +void PooledOdbcConnection::Release() +{ + if (pool_ != nullptr && dbc_ != SQL_NULL_HDBC) { + pool_->ReturnConnection(dbc_); + } + pool_ = nullptr; + dbc_ = SQL_NULL_HDBC; +} + +MysqlOdbcConnectionPool::MysqlOdbcConnectionPool(DatabaseConfig config) : config_(std::move(config)){} +MysqlOdbcConnectionPool::~MysqlOdbcConnectionPool() +{ + std::lock_guard lk(mu_); + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } +} + +std::optional MysqlOdbcConnectionPool::Initialize() +{ + auto cleanup_partial = [this]() { + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } + }; + + SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &henv_); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLAllocHandle(ENV) failed"); + } + + rc = SQLSetEnvAttr(henv_, SQL_ATTR_ODBC_VERSION, (SQLPOINTER)SQL_OV_ODBC3, 0); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + return std::string("SQLSetEnvAttr failed"); + } + + const std::size_t pool_size = config_.max_pool_connections; + all_handles_.reserve(pool_size); + + for (std::size_t i = 0; i < pool_size; ++i) { + SQLHDBC dbc = SQL_NULL_HDBC; + rc = SQLAllocHandle(SQL_HANDLE_DBC, henv_, &dbc); + if (!SQL_SUCCEEDED(rc)) { + cleanup_partial(); + return std::string("SQLAllocHandle(DBC) failed"); + } + + if (!config_.database_ip.empty()) { + const std::string conn_str = BuildMySqlDriverConnectString(config_); + SQLCHAR out_conn[1024]{}; + SQLSMALLINT out_conn_len = 0; + rc = SQLDriverConnect(dbc, nullptr, reinterpret_cast(const_cast(conn_str.data())), + SQL_NTS, out_conn, sizeof(out_conn), &out_conn_len, SQL_DRIVER_NOPROMPT); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLDriverConnect failed"); + } + } else { + rc = SQLConnect(dbc,reinterpret_cast(const_cast(config_.primary_dsn_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_password.data())), + SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLConnect failed"); + } + } + + all_handles_.push_back(dbc); + free_.push_back(dbc); + } + + return std::nullopt; +} + +PooledOdbcConnection MysqlOdbcConnectionPool::Acquire() +{ + std::unique_lock lk(mu_); + if (!cv_.wait_for(lk, kPoolAcquireTimeout, [this] { return !free_.empty(); })) { + return PooledOdbcConnection(); + } + SQLHDBC dbc = free_.front(); + free_.pop_front(); + return PooledOdbcConnection(this, dbc); +} + +void MysqlOdbcConnectionPool::ReturnConnection(SQLHDBC dbc) +{ + { + std::lock_guard lk(mu_); + free_.push_back(dbc); + } + cv_.notify_one(); +} + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool) +{ + g_global_mysql_odbc_pool.store(pool, std::memory_order_release); +} + +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool() +{ + return g_global_mysql_odbc_pool.load(std::memory_order_acquire); +} + +namespace { + + +constexpr const char kSqlBtModelsMaxTs[] = "SELECT UNIX_TIMESTAMP(MAX(update_timestamp)) FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1"; +constexpr const char kSqlBtModelsForDc[] = "SELECT campaign_id, model_name, feature_mapping, feature_sequence, applicable_campaigns FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1 AND dc_id = ? ORDER BY update_timestamp DESC"; +constexpr size_t kModelNameBuf = kMaxModelNameLen + 1; +constexpr size_t kFeatureMappingBuf = kMaxFeatureMappingJsonLen + 1; +constexpr size_t kFeatureSequenceBuf = kMaxFeatureSequenceLen + 1; +constexpr size_t kApplicableCampaignsBuf = kMaxApplicableCampaignsLen + 1; + +struct OdbcStmt { + SQLHSTMT h{SQL_NULL_HSTMT}; + explicit OdbcStmt(SQLHDBC dbc) + { + const SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_STMT, dbc, &h); + if (!SQL_SUCCEEDED(rc)) { + h = SQL_NULL_HSTMT; + } + } + ~OdbcStmt() + { + if (h != SQL_NULL_HSTMT) { + SQLFreeHandle(SQL_HANDLE_STMT, h); + } + } + OdbcStmt(const OdbcStmt&) = delete; + OdbcStmt& operator=(const OdbcStmt&) = delete; +}; + +void Trim(std::string* s) +{ + if (s == nullptr || s->empty()) { + return; + } + const auto not_space = [](unsigned char c) { return !std::isspace(c); }; + auto b = std::find_if(s->begin(), s->end(), not_space); + auto e = std::find_if(s->rbegin(), s->rend(), not_space).base(); + if (b >= e) { + s->clear(); + } else { + *s = std::string(b, e); + } +} + +void ToLowerInPlace(std::string* s) +{ + if (s == nullptr) { + return; + } + std::transform(s->begin(), s->end(), s->begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); +} + +std::string SqlCharBufferToString(const std::vector& buf, SQLLEN cb) +{ + if (cb == SQL_NULL_DATA) { + return {}; + } + const char* p = reinterpret_cast(buf.data()); + if (cb < 0) { + return std::string(p); + } + return std::string(p, static_cast(cb)); +} + +std::string JsonScalarToString(const rapidjson::Value& v) +{ + if (v.IsString()) { + return std::string(v.GetString(), v.GetStringLength()); + } + if (v.IsBool()) { + return v.GetBool() ? "true" : "false"; + } + if (v.IsInt()) { + return std::to_string(v.GetInt()); + } + if (v.IsUint()) { + return std::to_string(v.GetUint()); + } + if (v.IsInt64()) { + return std::to_string(v.GetInt64()); + } + if (v.IsUint64()) { + return std::to_string(v.GetUint64()); + } + if (v.IsDouble()) { + return std::to_string(v.GetDouble()); + } + if (v.IsNull()) { + return {}; + } + return {}; +} + +bool TryParseMappingInt64(const std::string& s, int64_t* out) +{ + if (out == nullptr || s.empty()) { + return false; + } + const char* begin = s.c_str(); + char* end = nullptr; + errno = 0; + const long long v = std::strtoll(begin, &end, 10); + if (errno != 0 || end != begin + static_cast(s.size())) { + return false; + } + *out = static_cast(v); + return true; +} + +std::optional FetchMaxUnixTimestampFromDbc(SQLHDBC dbc, const char* sql, int64_t* out_ts) +{ + *out_ts = 0; + OdbcStmt st(dbc); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + SQLRETURN rc = SQLExecDirect(st.h, reinterpret_cast(const_cast(sql)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecDirect failed"); + } + int64_t ts = 0; + SQLLEN cb_ts = 0; + rc = SQLBindCol(st.h, 1, SQL_C_SBIGINT, &ts, 0, &cb_ts); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol failed"); + } + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + return std::nullopt; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_ts != SQL_NULL_DATA) { + *out_ts = ts; + } + return std::nullopt; +} + +} // namespace + +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out) +{ + out->clear(); + std::size_t start = 0; + while (start < s.size()) { + const std::size_t comma = s.find(',', start); + std::string piece = (comma == std::string::npos) ? s.substr(start) : s.substr(start, comma - start); + Trim(&piece); + if (!piece.empty()) { + out->push_back(std::move(piece)); + } + if (comma == std::string::npos) { + break; + } + start = comma + 1; + } +} + +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out) +{ + out->clear(); + std::string t = s; + Trim(&t); + if (t.empty()) { + return std::nullopt; + } + if (t.size() == 2 && (t[0] == 'n' || t[0] == 'N') && (t[1] == 'a' || t[1] == 'A')) { + return std::nullopt; + } + + std::vector tokens; + SplitCommaSeparatedStrings(t, &tokens); + if (tokens.empty()) { + return std::nullopt; + } + + for (const auto& tok : tokens) { + char* endptr = nullptr; + const long v = std::strtol(tok.c_str(), &endptr, 10); + if (endptr == tok.c_str() || *endptr != '\0') { + return std::string("invalid campaign id token: ") + tok; + } + if (v < INT32_MIN || v > INT32_MAX) { + return std::string("campaign id out of int32_t range: ") + tok; + } + out->push_back(static_cast(v)); + } + return std::nullopt; +} + +int GetFeatureMappingIdx(const char* feature_name, const char* feature, const FeatureMappingTables* feature_mapping) +{ + if (feature_mapping == nullptr || feature_name == nullptr || feature == nullptr) { + return -1; + } + const auto outer = feature_mapping->find(feature_name); + if (outer == feature_mapping->end()) { + return -1; + } + const auto inner = outer->second.value_to_index.find(feature); + if (inner == outer->second.value_to_index.end()) { + return -1; + } + return inner->second; +} + +int GetFeatureMappingIdxForInt64(const char* feature_name, int64_t feature_value, const FeatureMappingTables* feature_mapping) { + if (feature_mapping == nullptr || feature_name == nullptr) { + return -1; + } + const auto outer = feature_mapping->find(feature_name); + if (outer == feature_mapping->end()) { + return -1; + } + const auto inner = outer->second.int_value_to_index.find(feature_value); + if (inner == outer->second.int_value_to_index.end()) { + return -1; + } + return inner->second; +} + +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error) +{ + out->clear(); + if (parse_error != nullptr) { + parse_error->clear(); + } + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + if (parse_error != nullptr) { + *parse_error = std::string("JSON parse error at offset ") + std::to_string(doc.GetErrorOffset()) + ": " + rapidjson::GetParseError_En(doc.GetParseError()); + } + return false; + } + if (!doc.IsObject()) { + if (parse_error != nullptr) { + *parse_error = "feature_mapping root must be a JSON object"; + } + return false; + } + + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + if (!it->name.IsString()) { + continue; + } + const std::string feature_name(it->name.GetString(), it->name.GetStringLength()); + if (feature_name.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "feature name exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1"; + } + return false; + } + if (!it->value.IsArray()) { + if (parse_error != nullptr) { + *parse_error = "feature '" + feature_name + "' value is not a JSON array"; + } + return false; + } + const rapidjson::Value& arr = it->value; + FeatureValueIndexMap table; + table.values.reserve(arr.Size()); + for (rapidjson::SizeType i = 0; i < arr.Size(); ++i) { + const std::string cell = JsonScalarToString(arr[i]); + if (cell.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "categorical value exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1 for feature '" + feature_name + "'"; + } + return false; + } + table.values.push_back(cell); + table.value_to_index[cell] = static_cast(i); + int64_t as_int = 0; + if (TryParseMappingInt64(cell, &as_int)) { + table.int_value_to_index[as_int] = static_cast(i); + } + } + (*out)[feature_name] = std::move(table); + } + return true; +} + +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts) +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + return FetchMaxUnixTimestampFromDbc(conn.handle(), kSqlBtModelsMaxTs, out_ts); +} + +namespace { + +bool TryMergeRowIntoCampaignMap(const LightgbmBtModelRow& row, CampaignToFeatureMappings& out) +{ + const CampaignBtModelBundle bundle{row.model_name, row.model_name_lower, row.feature_mapping, row.feature_sequence}; + + if (row.campaign_id != 0) { + if (out.find(row.campaign_id) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << row.campaign_id << " already exists in triton models map; skipping row"; +#endif + return false; + } + out[row.campaign_id] = bundle; + return true; + } + + if (row.applicable_campaigns.empty()) { + if (out.find(0) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign 0 already exists in triton models map; skipping row"; +#endif + return false; + } + out[0] = bundle; + return true; + } + + bool any_inserted = false; + for (int32_t cid : row.applicable_campaigns) { + if (out.find(cid) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << cid << " already exists in triton models map"; +#endif + continue; + } + out[cid] = bundle; + any_inserted = true; + } + return any_inserted; +} + +} // namespace + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map) +{ + out_campaign_map.clear(); + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + const int32_t dc_id = pool->Config().dc_id; + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + + OdbcStmt st(conn.handle()); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + + SQLRETURN rc = SQLPrepare(st.h, reinterpret_cast(const_cast(kSqlBtModelsForDc)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLPrepare failed"); + } + + SQLINTEGER dc_param = static_cast(dc_id); + rc = SQLBindParameter(st.h, 1, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, &dc_param, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindParameter failed"); + } + + rc = SQLExecute(st.h); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecute failed"); + } + + SQLINTEGER s_campaign_id = 0; + SQLLEN cb_campaign_id = 0; + std::vector model_buf(kModelNameBuf + 1, 0); + SQLLEN cb_model_name = 0; + std::vector mapping_buf(kFeatureMappingBuf + 1, 0); + SQLLEN cb_mapping = 0; + std::vector sequence_buf(kFeatureSequenceBuf + 1, 0); + SQLLEN cb_sequence = 0; + std::vector campaigns_buf(kApplicableCampaignsBuf + 1, 0); + SQLLEN cb_campaigns = 0; + + int col = 1; + rc = SQLBindCol(st.h, col++, SQL_C_SLONG, &s_campaign_id, 0, &cb_campaign_id); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(campaign_id) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, model_buf.data(), static_cast(model_buf.size()), &cb_model_name); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(model_name) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, mapping_buf.data(), static_cast(mapping_buf.size()), &cb_mapping); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_mapping) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, sequence_buf.data(), static_cast(sequence_buf.size()), &cb_sequence); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_sequence) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, campaigns_buf.data(), static_cast(campaigns_buf.size()), &cb_campaigns); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(applicable_campaigns) failed"); + } + + while (true) { + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + break; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_campaign_id == SQL_NULL_DATA) { + continue; + } + + LightgbmBtModelRow row; + row.campaign_id = static_cast(s_campaign_id); + + if (cb_model_name != SQL_NULL_DATA) { + row.model_name = SqlCharBufferToString(model_buf, cb_model_name); + Trim(&row.model_name); + row.model_name_lower = row.model_name; + ToLowerInPlace(&row.model_name_lower); + } + + std::string mapping_json; + if (cb_mapping != SQL_NULL_DATA) { + mapping_json = SqlCharBufferToString(mapping_buf, cb_mapping); + Trim(&mapping_json); + } + std::string parse_err; + if (!ParseFeatureMappingJson(mapping_json, &row.feature_mapping, &parse_err)) { + return std::string("feature_mapping JSON: ") + parse_err; + } + + std::string seq_str; + if (cb_sequence != SQL_NULL_DATA) { + seq_str = SqlCharBufferToString(sequence_buf, cb_sequence); + Trim(&seq_str); + } + SplitCommaSeparatedStrings(seq_str, &row.feature_sequence); + + std::string camp_str; + if (cb_campaigns != SQL_NULL_DATA) { + camp_str = SqlCharBufferToString(campaigns_buf, cb_campaigns); + Trim(&camp_str); + } + if (auto err = ParseApplicableCampaignIds(camp_str, &row.applicable_campaigns)) { + return err; + } + + if (!TryMergeRowIntoCampaignMap(row, out_campaign_map)) { + continue; + } + } + + return std::nullopt; +} + +namespace { + std::array g_triton_campaign_feature_mappings; + std::atomic g_triton_models_active{0}; + std::atomic g_triton_models_modification_time{0}; +} + +bool IsTritonModelsModified() +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return false; + } + + int retry_count = pool->Config().query_retry_count; + if (retry_count < 0) { + retry_count = 0; + } + + int64_t last_updated = 0; + std::optional err; + do { + err = FetchLightgbmBtModelsMaxUpdateUnixSeconds(&last_updated); + if (!err) { + break; + } + } while (err.has_value() && retry_count--); + + if (err.has_value()) { + return false; + } + + const uint32_t lu = static_cast(last_updated); + const uint32_t prev = g_triton_models_modification_time.load(std::memory_order_relaxed); + if (lu > prev) { + g_triton_models_modification_time.store(lu, std::memory_order_relaxed); + return true; + } + return false; +} + +std::optional UpdateTritonModelsData() +{ + if (!IsTritonModelsModified()) { + return std::nullopt; + } + + CampaignToFeatureMappings by_campaign; + if (auto err = FetchLightgbmBtModelsForDc(by_campaign)) { + return err; + } + + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + const int inactive = 1 - idx; + g_triton_campaign_feature_mappings[inactive] = std::move(by_campaign); + g_triton_models_active.store(inactive, std::memory_order_release); + return std::nullopt; +} + +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings() +{ + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + return &g_triton_campaign_feature_mappings[idx]; +} + +}} // namespace triton::server diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h new file mode 100644 index 0000000000..0c6044b96c --- /dev/null +++ b/src/mysql_odbc_connection_pool.h @@ -0,0 +1,171 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Connection pool over the MySQL ODBC driver using the ODBC API (unixODBC / +// iODBC). Build with -DTRITON_ENABLE_MYSQL_ODBC=ON and install unixodbc-dev +// (Debian/Ubuntu) plus a configured MySQL ODBC DSN. + +#pragma once + +#include "database_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +#include +#include + +namespace triton { namespace server { + +inline constexpr std::size_t kMaxModelNameLen = 100; +inline constexpr std::size_t kMaxFeatureMappingJsonLen = 16 * (1 << 20); +inline constexpr std::size_t kMaxFeatureSequenceLen = 1000; +inline constexpr std::size_t kMaxApplicableCampaignsLen = 4096; +inline constexpr std::size_t kMaxLightgbmUpdateTimestampLen = 256; +inline constexpr std::size_t kTritonFeatureMappingBuffSize = 256; +inline constexpr std::size_t kTritonFeatureMappingMaxTokenLen = + kTritonFeatureMappingBuffSize - 1; + +class MysqlOdbcConnectionPool; + +class PooledOdbcConnection { + public: + PooledOdbcConnection(); + ~PooledOdbcConnection(); + + PooledOdbcConnection(PooledOdbcConnection&& other) noexcept; + PooledOdbcConnection& operator=(PooledOdbcConnection&& other) noexcept; + + PooledOdbcConnection(const PooledOdbcConnection&) = delete; + PooledOdbcConnection& operator=(const PooledOdbcConnection&) = delete; + + SQLHDBC handle() const { return dbc_; } + explicit operator bool() const { return dbc_ != SQL_NULL_HDBC; } + + private: + friend class MysqlOdbcConnectionPool; + PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc); + + void Release(); + + MysqlOdbcConnectionPool* pool_{nullptr}; + SQLHDBC dbc_{SQL_NULL_HDBC}; +}; + +class MysqlOdbcConnectionPool { + public: + explicit MysqlOdbcConnectionPool(DatabaseConfig config); + ~MysqlOdbcConnectionPool(); + + MysqlOdbcConnectionPool(const MysqlOdbcConnectionPool&) = delete; + MysqlOdbcConnectionPool& operator=(const MysqlOdbcConnectionPool&) = delete; + + std::optional Initialize(); + + PooledOdbcConnection Acquire(); + + const DatabaseConfig& Config() const { return config_; } + + private: + friend class PooledOdbcConnection; + void ReturnConnection(SQLHDBC dbc); + + DatabaseConfig config_; + SQLHENV henv_{SQL_NULL_HENV}; + std::vector all_handles_; + std::deque free_; + std::mutex mu_; + std::condition_variable cv_; +}; + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool); +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool(); + +struct FeatureValueIndexMap { + std::vector values; + std::unordered_map value_to_index; + // Fast path for JSON numeric features (avoids snprintf per request row). + std::unordered_map int_value_to_index; +}; + +using FeatureMappingTables = std::unordered_map; + +// Legacy get_feature_mapping_idx: look up categorical index for `feature` +// (token string) under column `feature_name`. Returns -1 if any map or key is +// missing, or if `feature_name` / `feature` / `feature_mapping` is null. +int GetFeatureMappingIdx( + const char* feature_name, const char* feature, + const FeatureMappingTables* feature_mapping); + +// Look up categorical index when the request feature value is already numeric. +int GetFeatureMappingIdxForInt64( + const char* feature_name, int64_t feature_value, + const FeatureMappingTables* feature_mapping); + +// Loaded from `lightgbm_bt_models` per campaign_id (after merge rules). +struct CampaignBtModelBundle { + std::string model_name; + std::string model_name_lower; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; +}; + +using CampaignToFeatureMappings = std::unordered_map; + +struct LightgbmBtModelRow { + int32_t campaign_id{0}; + std::string model_name; + std::string model_name_lower; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; + std::vector applicable_campaigns; +}; + +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts); + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map); +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error); +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out_tokens); +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out_ids); +bool IsTritonModelsModified(); +std::optional UpdateTritonModelsData(); +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings(); + +}} // namespace triton::server + diff --git a/src/sagemaker_server.cc b/src/sagemaker_server.cc index 52074f2b9d..75e9eb8363 100644 --- a/src/sagemaker_server.cc +++ b/src/sagemaker_server.cc @@ -25,6 +25,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sagemaker_server.h" +#include "http_error_json.h" + namespace triton { namespace server { #define HTTP_RESPOND_IF_ERR(REQ, X) \ @@ -40,21 +42,6 @@ namespace triton { namespace server { namespace { -void -EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) -{ - const char* message = TRITONSERVER_ErrorMessage(err); - - triton::common::TritonJson::Value response( - triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - TRITONSERVER_Error* EVBufferToJson( triton::common::TritonJson::Value* document, evbuffer_iovec* v, int* v_idx, diff --git a/src/sagemaker_server.h b/src/sagemaker_server.h index dcd40e66ac..ffb94322df 100644 --- a/src/sagemaker_server.h +++ b/src/sagemaker_server.h @@ -53,10 +53,12 @@ class SagemakerAPIServer : public HTTPAPIServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) : InferRequestClass( server, req, response_compression_type, triton_request, - shm_manager) + shm_manager, pause_http_request, register_fini_cancel_hook) { } using InferRequestClass::InferResponseComplete; @@ -123,12 +125,13 @@ class SagemakerAPIServer : public HTTPAPIServer { std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) - override + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) override { return std::unique_ptr(new SagemakeInferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, diff --git a/src/transform.cc b/src/transform.cc new file mode 100644 index 0000000000..2956570458 --- /dev/null +++ b/src/transform.cc @@ -0,0 +1,510 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include "mysql_odbc_connection_pool.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::unordered_set g_ready_model_names; +std::atomic g_ready_models_valid{false}; + +int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, const triton::server::FeatureMappingTables* tables) { + if (tables == nullptr) { + return -1; + } + if (v.IsString()) { + return triton::server::GetFeatureMappingIdx(feature_name, v.GetString(), tables); + } + if (v.IsInt()) { + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(v.GetInt()), tables); + } + if (v.IsUint()) { + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(v.GetUint()), tables); + } + if (v.IsInt64()) { + return triton::server::GetFeatureMappingIdxForInt64(feature_name, v.GetInt64(), tables); + } + if (v.IsUint64()) { + const uint64_t uv = v.GetUint64(); + if (uv > static_cast(INT64_MAX)) { + char num_buf[32]; + const int n = std::snprintf(num_buf, sizeof(num_buf), "%llu", static_cast(uv)); + if (n <= 0 || static_cast(n) >= sizeof(num_buf)) { + return -1; + } + return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); + } + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(uv), tables); + } + return -1; +} + +bool IsCampLevelFeature(const std::string& feature) { + return feature == TRITON_BT_FEATURE_COOKIE || feature == TRITON_BT_FEATURE_RNK || feature == TRITON_BT_FEATURE_CAMPID; +} + +bool UsesRawNumericFeature(const std::string& feature) { + return (feature == TRITON_BT_FEATURE_UID) || (feature == TRITON_BT_FEATURE_VIDEO_VPW) || (feature == TRITON_BT_FEATURE_VIDEO_VPH) || (feature == TRITON_BT_FEATURE_MOBILEID) || (feature == TRITON_BT_FEATURE_VIEW) || (feature == TRITON_BT_FEATURE_COOKIE); +} + +TRITONSERVER_Error* FillRawNumericFeature(const std::string& feature, const rapidjson::Value& v, float* out) { + if (v.IsInt()) { + *out = static_cast(v.GetInt()); + } else if (v.IsUint()) { + *out = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + *out = static_cast(v.GetInt64()); + } else if (v.IsUint64()) { + *out = static_cast(v.GetUint64()); + } else if (v.IsDouble()) { + *out = static_cast(v.GetDouble()); + } else { + const std::string msg = std::string("feature '") + feature + "' must be a JSON number for raw passthrough"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg.c_str()); + } + return nullptr; +} + +TRITONSERVER_Error* BuildImpBaseRow(const rapidjson::Value& imp, const std::vector& feature_sequence, const triton::server::FeatureMappingTables& tables, std::vector* out_row) { + out_row->assign(feature_sequence.size(), 0.0f); + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + if (feature == TRITON_BT_FEATURE_ADSIZE || IsCampLevelFeature(feature)) { + continue; + } + + const char* fkey = feature.c_str(); + if (!imp.HasMember(fkey)) { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + const rapidjson::Value& v = imp[fkey]; + + TRITONSERVER_Error* err = nullptr; + if (UsesRawNumericFeature(feature)) { + err = FillRawNumericFeature(feature, v, &(*out_row)[fi]); + } else { + (*out_row)[fi] = static_cast(FeatureIdxFromJsonValue(feature.c_str(), v, &tables)); + } + if (err != nullptr) { + return err; + } + } + return nullptr; +} + +TRITONSERVER_Error* FillCampFeaturesInRow(const rapidjson::Value& camp, int32_t campaign_id, const std::vector& feature_sequence, const triton::server::FeatureMappingTables& tables, std::vector* row) { + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + if (feature == TRITON_BT_FEATURE_ADSIZE) { + continue; + } + if (!IsCampLevelFeature(feature)) { + continue; + } + + const rapidjson::Value* src = nullptr; + if (feature == TRITON_BT_FEATURE_CAMPID) { + src = &camp[TRITON_BT_JSON_CID]; + } else if (camp.HasMember(feature.c_str())) { + src = &camp[feature.c_str()]; + } else { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + + TRITONSERVER_Error* err = nullptr; + if (UsesRawNumericFeature(feature)) { + err = FillRawNumericFeature(feature, *src, &(*row)[fi]); + } else { + (*row)[fi] = static_cast(FeatureIdxFromJsonValue(feature.c_str(), *src, &tables)); + } + if (err != nullptr) { + return err; + } + } + return nullptr; +} + +TRITONSERVER_Error* RefreshReadyModelNamesInto(std::unordered_set* out, TRITONSERVER_Server* server) { + if (out == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output set pointer is null"); + } + out->clear(); + + TRITONSERVER_Message* message = nullptr; + TRITONSERVER_Error* err = TRITONSERVER_ServerModelIndex(server, TRITONSERVER_INDEX_FLAG_READY, &message); + if (err != nullptr) { + return err; + } + + const char* buffer = nullptr; + size_t byte_size = 0; + err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); + if (err != nullptr) { + TRITONSERVER_MessageDelete(message); + return err; + } + const std::string index_json(buffer, byte_size); + TRITONSERVER_MessageDelete(message); + + rapidjson::Document index_doc; + index_doc.Parse(index_json.data(), index_json.size()); + if (index_doc.HasParseError()) { + const std::string parse_err = std::string("failed to parse model index JSON: ") + rapidjson::GetParseError_En(index_doc.GetParseError()) + " at offset " + std::to_string(index_doc.GetErrorOffset()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, parse_err.c_str()); + } + if (!index_doc.IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "model index JSON root is not an array"); + } + + out->reserve(static_cast(index_doc.Size())); + for (rapidjson::SizeType i = 0; i < index_doc.Size(); ++i) { + const rapidjson::Value& o = index_doc[i]; + if (!o.IsObject() || !o.HasMember("name")) { + continue; + } + const rapidjson::Value& n = o["name"]; + if (!n.IsString()) { + continue; + } + out->emplace(n.GetString()); + } + + return nullptr; +} + +} // namespace + +namespace triton { namespace server { + +namespace { + +struct ModelSlotBuild { + size_t feature_count{0}; + int adsize_idx{-1}; + std::vector tensor; + std::vector routes; + std::string original_model_name; +}; + +int AdsizeFeatureIndex(const std::vector& feature_sequence) +{ + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + if (feature_sequence[fi] == TRITON_BT_FEATURE_ADSIZE) { + return static_cast(fi); + } + } + return -1; +} + +const CampaignBtModelBundle* LookupCampaignBundle(const CampaignToFeatureMappings* cmap, int32_t campaign_id) +{ + auto it = cmap->find(campaign_id); + if (it == cmap->end()) { + it = cmap->find(0); + } + if (it == cmap->end()) { + return nullptr; + } + return &it->second; +} + +size_t CountCampInferRows(const rapidjson::Value& camp, int adsize_idx) +{ + if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { + return camp[TRITON_BT_FEATURE_ADSIZE].Size(); + } + return 0; +} + +TRITONSERVER_Error* CheckModelReadyFromSnapshot(const std::string& model_name, const std::string& original_model_name, std::unordered_set* verified_models) +{ + if (verified_models->find(model_name) != verified_models->end()) { + return nullptr; + } + + const std::unordered_set* ready = ActiveReadyModelNames(); + if (ready == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "ready model snapshot is not initialized"); + } + + if (ready->find(model_name) != ready->end() || + (!original_model_name.empty() && ready->find(original_model_name) != ready->end())) { + verified_models->insert(model_name); + return nullptr; + } + + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); +} + +void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) +{ + const size_t nbytes = row.size() * sizeof(float); + if (nbytes == 0) { + return; + } + const char* bytes = reinterpret_cast(row.data()); + tensor->insert(tensor->end(), bytes, bytes + nbytes); +} + +} // namespace + +TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server) { + if (server == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "server pointer is null"); + } + if (g_ready_models_valid.load(std::memory_order_acquire)) { + return nullptr; + } + TRITONSERVER_Error* err = RefreshReadyModelNamesInto(&g_ready_model_names, server); + if (err != nullptr) { + return err; + } + g_ready_models_valid.store(true, std::memory_order_release); + return nullptr; +} + +const std::unordered_set* ActiveReadyModelNames() { + if (!g_ready_models_valid.load(std::memory_order_acquire)) { + return nullptr; + } + return &g_ready_model_names; +} + +TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITONSERVER_Server* server, std::vector* out_slots, ImpRoutingTable* out_routing) { + if (out_slots == nullptr || server == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); + } + if (!doc.IsObject() || !doc.HasMember(TRITON_BT_JSON_IMPS) || !doc[TRITON_BT_JSON_IMPS].IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array"); + } + + const CampaignToFeatureMappings* cmap = ActiveCampaignToFeatureMappings(); + if (cmap == nullptr || cmap->empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); + } + + const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; + + std::unordered_map rows_per_model; + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { + const rapidjson::Value& imp = imps[ii]; + if (!imp.IsObject()) { + continue; + } + if (!imp.HasMember(TRITON_BT_JSON_CAMPS) || !imp[TRITON_BT_JSON_CAMPS].IsArray()) { + continue; + } + const rapidjson::Value& camps = imp[TRITON_BT_JSON_CAMPS]; + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { + const rapidjson::Value& camp = camps[ci]; + if (!camp.IsObject() || !camp.HasMember(TRITON_BT_JSON_CID) || !camp[TRITON_BT_JSON_CID].IsInt()) { + continue; + } + const CampaignBtModelBundle* bundle = LookupCampaignBundle(cmap, camp[TRITON_BT_JSON_CID].GetInt()); + if (bundle == nullptr) { + continue; + } + const int adsize_idx = AdsizeFeatureIndex(bundle->feature_sequence); + rows_per_model[bundle->model_name_lower] += CountCampInferRows(camp, adsize_idx); + } + } + + std::unordered_set verified_ready_models; + std::unordered_map slots_by_model; + const std::string* cached_slot_model_name = nullptr; + ModelSlotBuild* cached_slot_build = nullptr; + TRITONSERVER_Error* err = nullptr; + + out_slots->clear(); + if (out_routing != nullptr) { + out_routing->imp_count = 0; + out_routing->slots.clear(); + } + + std::vector scratch_row; + std::unordered_map> imp_base_by_model; + + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { + const rapidjson::Value& imp = imps[ii]; + if (!imp.IsObject()) { + continue; + } + if (!imp.HasMember(TRITON_BT_JSON_CAMPS) || !imp[TRITON_BT_JSON_CAMPS].IsArray() || imp[TRITON_BT_JSON_CAMPS].Size() == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each impression must include a non-empty camps array"); + } + const rapidjson::Value& camps = imp[TRITON_BT_JSON_CAMPS]; + imp_base_by_model.clear(); + + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { + const rapidjson::Value& camp = camps[ci]; + if (!camp.IsObject() || !camp.HasMember(TRITON_BT_JSON_CID) || !camp[TRITON_BT_JSON_CID].IsInt()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each camp must be an object with integer 'cid'"); + } + const int32_t campaign_id = camp[TRITON_BT_JSON_CID].GetInt(); + + auto cmap_it = cmap->find(campaign_id); + if (cmap_it == cmap->end()) { + cmap_it = cmap->find(0); + } + if (cmap_it == cmap->end()) { + const std::string unknown_campaign = std::string("unknown campaign_id ") + std::to_string(campaign_id); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, unknown_campaign.c_str()); + } + const std::string& model_name = cmap_it->second.model_name_lower; + const std::string& original_model_name = cmap_it->second.model_name; + const std::vector& feature_sequence = cmap_it->second.feature_sequence; + const FeatureMappingTables& tables = cmap_it->second.feature_mapping; + + err = CheckModelReadyFromSnapshot(model_name, original_model_name, &verified_ready_models); + if (err != nullptr) { + return err; + } + + ModelSlotBuild* slot_build = nullptr; + if (cached_slot_model_name != nullptr && model_name == *cached_slot_model_name) { + slot_build = cached_slot_build; + } else { + auto slot_it = slots_by_model.find(model_name); + if (slot_it == slots_by_model.end()) { + slot_it = slots_by_model.emplace(model_name, ModelSlotBuild{}).first; + slot_it->second.original_model_name = original_model_name; + slot_it->second.adsize_idx = AdsizeFeatureIndex(feature_sequence); + const auto row_est = rows_per_model.find(model_name); + if (row_est != rows_per_model.end() && !feature_sequence.empty()) { + slot_it->second.tensor.reserve(row_est->second * feature_sequence.size() * sizeof(float)); + } + } + cached_slot_model_name = &slot_it->first; + cached_slot_build = &slot_it->second; + slot_build = cached_slot_build; + } + const size_t feature_count = feature_sequence.size(); + if (slot_build->feature_count == 0) { + slot_build->feature_count = feature_count; + } + else if (slot_build->feature_count != feature_count) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); + } + + auto base_it = imp_base_by_model.find(model_name); + if (base_it == imp_base_by_model.end()) { + std::vector base_row; + err = BuildImpBaseRow(imp, feature_sequence, tables, &base_row); + if (err != nullptr) { + return err; + } + base_it = imp_base_by_model.emplace(model_name, std::move(base_row)).first; + } + + if (scratch_row.size() != feature_count) { + scratch_row.resize(feature_count); + } + std::memcpy(scratch_row.data(), base_it->second.data(), feature_count * sizeof(float)); + err = FillCampFeaturesInRow(camp, campaign_id, feature_sequence, tables, &scratch_row); + if (err != nullptr) { + return err; + } + + const int adsize_idx = slot_build->adsize_idx; + + if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { + const rapidjson::Value& adsize = camp[TRITON_BT_FEATURE_ADSIZE]; + for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { + const rapidjson::Value& adsize_item = adsize[ai]; + scratch_row[static_cast(adsize_idx)] = static_cast(FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables)); + AppendFloatRowToTensor(&slot_build->tensor, scratch_row); + if (out_routing != nullptr) { + slot_build->routes.push_back(ImpRouteRow{static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); + } + } + } + } + } + + std::vector model_names; + model_names.reserve(slots_by_model.size()); + for (const auto& kv : slots_by_model) { + model_names.push_back(kv.first); + } + std::sort(model_names.begin(), model_names.end()); + + out_slots->reserve(model_names.size()); + if (out_routing != nullptr) { + out_routing->imp_count = static_cast(imps.Size()); + out_routing->slots.reserve(model_names.size()); + } + + for (const std::string& model_name : model_names) { + auto it = slots_by_model.find(model_name); + if (it == slots_by_model.end()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "internal buffer map inconsistency"); + } + ModelSlotBuild& built = it->second; + if (built.feature_count == 0 || built.tensor.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "empty model buffer after transform"); + } + if (built.tensor.size() % (built.feature_count * sizeof(float)) != 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); + } + + ImpsInferSlot slot; + slot.model_name = model_name; + slot.original_model_name = built.original_model_name.empty() ? model_name : built.original_model_name; + slot.model_version = -1; + slot.feature_count = built.feature_count; + slot.rows = built.tensor.size() / (built.feature_count * sizeof(float)); + slot.input_tensor = std::move(built.tensor); + out_slots->push_back(std::move(slot)); + + if (out_routing != nullptr) { + out_routing->slots.push_back(std::move(built.routes)); + } + } + + return nullptr; +} + +}} // namespace triton::server +#endif // TRITON_ENABLE_MYSQL_ODBC diff --git a/src/transform.h b/src/transform.h new file mode 100644 index 0000000000..c11dafd095 --- /dev/null +++ b/src/transform.h @@ -0,0 +1,99 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +#include "triton/core/tritonserver.h" + +namespace triton { namespace server { + +#ifdef TRITON_ENABLE_MYSQL_ODBC + +#define TRITON_BT_FEATURE_ADSIZE "adsize" +#define TRITON_BT_FEATURE_COOKIE "cookie" +#define TRITON_BT_FEATURE_RNK "rnk" +#define TRITON_BT_FEATURE_CAMPID "campid" +#define TRITON_BT_JSON_IMPS "imps" +#define TRITON_BT_JSON_CAMPS "camps" +#define TRITON_BT_JSON_CID "cid" +#define TRITON_BT_FEATURE_UID "uid" +#define TRITON_BT_FEATURE_VIDEO_VPW "video_vpw" +#define TRITON_BT_FEATURE_VIDEO_VPH "video_vph" +#define TRITON_BT_FEATURE_MOBILEID "mobileid" +#define TRITON_BT_FEATURE_VIEW "view" + +// Per batched infer row when folding multi_infer results back to imps/camps. +struct ImpRouteRow { + int imp_idx{0}; + int camp_idx{0}; + int adsize_idx{0}; + int32_t cid{0}; +}; + +// In-memory routing for imps-shaped requests (not serialized into multi_infer JSON). +struct ImpRoutingTable { + int imp_count{0}; + // One vector per multi_infer request slot (sorted model name order). + std::vector> slots; +}; + +// One model's imps feature matrix ready for direct TRITONSERVER_InferenceRequest +// fill (replaces per-slot infer JSON for the imps fast path). +struct ImpsInferSlot { + std::string model_name; + std::string original_model_name; + int64_t model_version{0}; + // Row-major FP32 tensor bytes (rows * feature_count * sizeof(float)). + std::vector input_tensor; + size_t rows{0}; + size_t feature_count{0}; +}; + +constexpr const char* kImpsInputTensorName = "input__0"; +constexpr const char* kImpsOutputTensorName = "output__0"; + +// Populate the ready-model snapshot once at process startup (after models are loaded). +TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server); + +// Lock-free read of the snapshot initialized by InitializeReadyModelNames. +const std::unordered_set* ActiveReadyModelNames(); + +// Feature mapping + FP32 tensor build for POST /v2/predict imps requests. +TRITONSERVER_Error* GenerateImpsInferSlots( + const rapidjson::Document& doc, TRITONSERVER_Server* server, + std::vector* out_slots, + ImpRoutingTable* out_routing = nullptr); + +// Populates TRITONSERVER_InferenceRequest from a slot: HTTPAPIServer::FillImpsTritonRequest +// in http_server.cc (requires InferRequestClass for input lifetime and output alloc). + +#endif // TRITON_ENABLE_MYSQL_ODBC + +}} // namespace triton::server