From eb2e8926338c18f2d2b00b06f3ab84007dc8d76e Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Tue, 16 Jun 2026 17:53:17 +0530 Subject: [PATCH 01/14] Initial commit --- .../{pre-commit.yaml => pre-commit.yml} | 37 +- .gitignore | 4 + .pre-commit-config.yaml | 8 +- CMakeLists.txt | 20 + Dockerfile | 109 +++ Dockerfile.pubmatic-bt-tritonserver | 52 ++ build.py | 57 +- config/database_config.sample.json | 13 + docs/client_guide/openai_readme.md | 345 ++++++++- .../simple_identity/1/model.py | 26 + src/CMakeLists.txt | 84 +++ src/database_config.cc | 204 +++++ src/database_config.h | 56 ++ src/http_server.cc | 189 ++--- src/http_server.h | 37 +- src/http_server_macros.h | 41 ++ src/main.cc | 126 ++++ src/multi_infer.cc | 635 ++++++++++++++++ src/mysql_odbc_connection_pool.cc | 697 ++++++++++++++++++ src/mysql_odbc_connection_pool.h | 163 ++++ src/sagemaker_server.h | 13 +- src/transform.cc | 349 +++++++++ src/transform.h | 70 ++ 23 files changed, 3226 insertions(+), 109 deletions(-) rename .github/workflows/{pre-commit.yaml => pre-commit.yml} (60%) create mode 100644 Dockerfile create mode 100644 Dockerfile.pubmatic-bt-tritonserver create mode 100644 config/database_config.sample.json mode change 120000 => 100644 docs/client_guide/openai_readme.md create mode 100644 src/database_config.cc create mode 100644 src/database_config.h create mode 100644 src/http_server_macros.h create mode 100644 src/multi_infer.cc create mode 100644 src/mysql_odbc_connection_pool.cc create mode 100644 src/mysql_odbc_connection_pool.h create mode 100644 src/transform.cc create mode 100644 src/transform.h diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yml similarity index 60% rename from .github/workflows/pre-commit.yaml rename to .github/workflows/pre-commit.yml index 6c33d435c9..3e0db73aaa 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yml @@ -1,4 +1,8 @@ -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +======== +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +>>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,6 +28,28 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py +import json + +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """This model always returns the input that it has received.""" + + def initialize(self, args): + self.model_config = json.loads(args["model_config"]) + + def execute(self, requests): + """This function is called on inference request.""" + + responses = [] + for request in requests: + in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor_0 = pb_utils.Tensor("OUTPUT0", in_0.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor_0])) + return responses +======== name: pre-commit on: @@ -31,15 +57,16 @@ on: jobs: pre-commit: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5.0.0 with: fetch-depth: 2 - name: Get modified files id: modified-files run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/setup-python@v6.0.0 + - uses: pre-commit/action@v3.0.1 with: extra_args: --files ${{ steps.modified-files.outputs.modified_files }} +>>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml diff --git a/.gitignore b/.gitignore index bce94b6830..01abc6eaab 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,12 @@ test_results.txt artifacts cprofile *.prof +.venv +**/.venv # Test exclusions qa/L0_openai/openai tensorrtllm_models +tensorrtllm_mistral_models/ custom_tokenizer +replace-artifacts/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 663a36d631..cd6320fe0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: -- repo: https://github.com/timothycrosley/isort +- repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: - id: isort @@ -36,7 +36,7 @@ repos: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.3.0 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] @@ -57,7 +57,7 @@ repos: # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d03d85665..de1e229c2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,21 @@ cmake_minimum_required(VERSION 3.18) +# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5). +# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a +# separate CMake invocation; that step does not inherit -D variables from the top +# level unless they are also in the environment. If configure fails inside libevent +# with "Compatibility with CMake < 3.5 has been removed", use either: +# export CMAKE_POLICY_VERSION_MINIMUM=3.5 # before cmake AND cmake --build +# or install/use CMake 3.28.x–3.31.x for this build. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + message( + STATUS + "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export " + "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build " + "(see docs/customization_guide/build.md).") +endif() + project(tritonserver LANGUAGES C CXX) include(CMakeDependentOption) @@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) +option(TRITON_ENABLE_MYSQL_ODBC + "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)" + OFF) + # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) @@ -261,6 +280,7 @@ ExternalProject_Add(triton-server -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} + -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..27454c0d38 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Derivative image: start from an official Triton server image and replace only +# the tritonserver executable and libtritonserver.so. Backends, Python wheels, +# and the rest of the filesystem stay unchanged from the base image. +# +# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX): +# install/bin/tritonserver +# install/lib/libtritonserver.so +# +# Prepare artifacts (from repository root): +# mkdir -p replace-artifacts +# cp install/bin/tritonserver replace-artifacts/ +# cp install/lib/libtritonserver.so replace-artifacts/ +# cp /etc/odbc.ini replace-artifacts/odbc.ini # optional if databaseIp is set (DSN unused) +# Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for +# optional JSON field "odbcDriverName" if the default fails. +# +# Build (from repository root; match your Triton tag, e.g. r25.03): +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \ +# -t tritonserver:25.03-custom . +# +# CPU-only base example: +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \ +# -t tritonserver:25.03-custom-cpu . +# +# If your base image stores libtritonserver.so under lib64: +# --build-arg TRITON_LIB_SUBDIR=lib64 +# +# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.: +# --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04 +# +# Run (mount a model repository and optionally override DM config): +# docker run --rm --gpus=all \ +# -p8000:8000 -p8001:8001 -p8002:8002 \ +# -v /path/to/model_repo:/models:ro \ +# -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \ +# tritonserver:25.03-custom \ +# tritonserver --model-repository=/models +# +# For CPU-only, drop --gpus=all and use a CPU/min base image. + +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 + +FROM ${BASE_IMAGE} + +ARG TRITON_INSTALL_PREFIX=/opt/tritonserver +ARG TRITON_LIB_SUBDIR=lib + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe +# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc. +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in +# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under +# /usr/lib//odbc/ only. unixODBC then fails with "Can't open lib +# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/. +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json). +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro) +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json + +# Paths relative to the build context (repository root when building with ".") +COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver +COPY replace-artifacts/libtritonserver.so \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + +# Match ownership used by generated Triton Dockerfiles (triton-server uid) +RUN chown 1000:1000 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ + && chmod 755 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + diff --git a/Dockerfile.pubmatic-bt-tritonserver b/Dockerfile.pubmatic-bt-tritonserver new file mode 100644 index 0000000000..340c3b27b5 --- /dev/null +++ b/Dockerfile.pubmatic-bt-tritonserver @@ -0,0 +1,52 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Extend the image produced by build.py (`docker build -t tritonserver`) with +# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files. +# Build context: repository root (same as build.py final docker build). +# +# Prerequisites: +# - Image `tritonserver` must exist. +# - replace-artifacts/odbc.ini +# - replace-artifacts/triton-dmconfig.json +# +# Example: +# docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver . + +FROM tritonserver + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json diff --git a/build.py b/build.py index 5a0f96413a..6ea96f5218 100755 --- a/build.py +++ b/build.py @@ -498,6 +498,7 @@ def core_cmake_args(components, backends, cmake_dir, install_dir): cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends)) cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends)) + cargs.append(cmake_core_enable("TRITON_ENABLE_MYSQL_ODBC", FLAGS.enable_mysql_odbc)) cargs += cmake_core_extra_args() cargs.append(cmake_dir) @@ -902,6 +903,12 @@ def install_dcgm_libraries(dcgm_version, target_machine): def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): + buildbase_odbc_layer = "" + if FLAGS.enable_mysql_odbc: + buildbase_odbc_layer = """ +RUN yum install -y unixODBC-devel +""" + df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} @@ -959,6 +966,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): xz-devel \\ zlib-devel """ + df += buildbase_odbc_layer if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"): df += """ RUN curl -k -s -L https://github.com/ccache/ccache/archive/refs/tags/v4.10.2.tar.gz -o /tmp/ccache.tar.gz \\ @@ -1051,6 +1059,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): SHELL ["cmd", "/S", "/C"] """ else: + mysql_odbc_line = ( + " unixodbc-dev \\\n" if FLAGS.enable_mysql_odbc else "" + ) df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive @@ -1101,7 +1112,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): libarchive-dev \\ libxml2-dev \\ libnuma-dev \\ - wget \\ +""" + df += mysql_odbc_line + df += """ wget \\ && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ @@ -1904,6 +1917,34 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(finalargs, check_exitcode=True) + if ( + FLAGS.pubmatic_bt_tritonserver_image + and target_platform() != "windows" + and target_platform() != "rhel" + ): + docker_script.blankln() + docker_script.commentln(8) + docker_script.comment( + "Pubmatic BT image: extend tritonserver with MySQL ODBC and config files" + ) + docker_script.comment( + "Uses Dockerfile.pubmatic-bt-tritonserver; requires replace-artifacts/" + ) + docker_script.comment() + pubmatic_args = [ + "docker", + "build", + "-t", + "pubmatic-bt-tritonserver", + "-f", + os.path.join( + THIS_SCRIPT_DIR, "Dockerfile.pubmatic-bt-tritonserver" + ), + ".", + ] + docker_script.cwd(THIS_SCRIPT_DIR) + docker_script.cmd(pubmatic_args, check_exitcode=True) + # # CI base image... tritonserver_cibase # @@ -2616,6 +2657,20 @@ def enable_all(): required=False, help="Enable ARM MALI GPU support.", ) + parser.add_argument( + "--enable-mysql-odbc", + action="store_true", + required=False, + help="Build tritonserver with MySQL ODBC connection pool. For host builds install unixodbc-dev (Debian/Ubuntu) or unixODBC-devel (RHEL). Container builds add these when this flag is set.", + ) + parser.add_argument( + "--pubmatic-bt-tritonserver-image", + action="store_true", + required=False, + help='After building image "tritonserver", also build "pubmatic-bt-tritonserver" ' + "(Dockerfile.pubmatic-bt-tritonserver: ODBC driver + replace-artifacts/odbc.ini and " + "replace-artifacts/triton-dmconfig.json). Ubuntu/Debian-based container builds only.", + ) parser.add_argument( "--min-compute-capability", type=str, diff --git a/config/database_config.sample.json b/config/database_config.sample.json new file mode 100644 index 0000000000..db12e70189 --- /dev/null +++ b/config/database_config.sample.json @@ -0,0 +1,13 @@ +{ + "databaseIp" : "10.0.0.1", + "databasePort" : 3306, + "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver", + "primaryDSNName" : "PrimaryCentralisedMySQL", + "secondaryDSNName" : "SecondaryCentralisedMySQL", + "dsnUserName" : "kdbuser", + "dsnUserPassword" : "REPLACE_WITH_SECRET", + "queryRetryCount": 3, + "dcId": 1, + "minPoolConnections": 2, + "maxPoolConnections": 5 +} diff --git a/docs/client_guide/openai_readme.md b/docs/client_guide/openai_readme.md deleted file mode 120000 index 05ca8a99c5..0000000000 --- a/docs/client_guide/openai_readme.md +++ /dev/null @@ -1 +0,0 @@ -../../python/openai/README.md \ No newline at end of file diff --git a/docs/client_guide/openai_readme.md b/docs/client_guide/openai_readme.md new file mode 100644 index 0000000000..53a9f461f4 --- /dev/null +++ b/docs/client_guide/openai_readme.md @@ -0,0 +1,344 @@ + +# OpenAI-Compatible Frontend for Triton Inference Server (Beta) + +> [!NOTE] +> The OpenAI-Compatible API is currently in BETA. Its features and functionality +> are subject to change as we collect feedback. We're excited to hear any thoughts +> you have and what features you'd like to see! + +## Pre-requisites + +1. Docker + NVIDIA Container Runtime +2. A correctly configured `HF_TOKEN` for access to HuggingFace models. + - The current examples and testing primarily use the + [`meta-llama/Meta-Llama-3.1-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) + model, but you can manually bring your own models and adjust accordingly. + +## VLLM + +1. Launch the container and install dependencies: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 +``` + +2. Launch the OpenAI-compatible Triton Inference Server: +```bash +cd /opt/tritonserver/python/openai + +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository tests/vllm_models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +
+Example output + +``` +... ++-----------------------+---------+--------+ +| Model | Version | Status | ++-----------------------+---------+--------+ +| llama-3.1-8b-instruct | 1 | READY | <- Correct Model Loaded in Triton ++-----------------------+---------+--------+ +... +Found model: name='llama-3.1-8b-instruct', backend='vllm' +[WARNING] Adding CORS for the following origins: ['http://localhost'] +INFO: Started server process [126] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully +``` + +
+ +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-6930b296-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": + { + "content": "This is only a test.", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1727679085, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +4. Send a `/v1/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "prompt": "Machine learning is" +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-d51df75c-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "text": " a field of computer science that focuses on developing algorithms that allow computers to learn from" + } + ], + "created": 1727679266, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "text_completion", + "usage": null +} +``` + +
+ +5. Benchmark with `genai-perf`: +- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) +- Or try using genai-perf from the [SDK container](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) + +```bash +MODEL="llama-3.1-8b-instruct" +TOKENIZER="meta-llama/Meta-Llama-3.1-8B-Instruct" +genai-perf profile \ + --model ${MODEL} \ + --tokenizer ${TOKENIZER} \ + --service-kind openai \ + --endpoint-type chat \ + --url localhost:9000 \ + --streaming +``` + +
+Example output + +``` +2024-10-14 22:43 [INFO] genai_perf.parser:82 - Profiling these models: llama-3.1-8b-instruct +2024-10-14 22:43 [INFO] genai_perf.wrapper:163 - Running Perf Analyzer : 'perf_analyzer -m llama-3.1-8b-instruct --async --input-data artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/inputs.json -i http --concurrency-range 1 --endpoint v1/chat/completions --service-kind openai -u localhost:9000 --measurement-interval 10000 --stability-percentage 999 --profile-export-file artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export.json' + NVIDIA GenAI-Perf | LLM Metrics +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ +┃ Statistic ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p75 ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ +│ Time to first token (ms) │ 71.66 │ 64.32 │ 86.52 │ 76.13 │ 74.92 │ 73.26 │ +│ Inter token latency (ms) │ 18.47 │ 18.25 │ 18.72 │ 18.67 │ 18.61 │ 18.53 │ +│ Request latency (ms) │ 348.00 │ 274.60 │ 362.27 │ 355.41 │ 352.29 │ 350.66 │ +│ Output sequence length │ 15.96 │ 12.00 │ 16.00 │ 16.00 │ 16.00 │ 16.00 │ +│ Input sequence length │ 549.66 │ 548.00 │ 551.00 │ 550.00 │ 550.00 │ 550.00 │ +│ Output token throughput (per sec) │ 45.84 │ N/A │ N/A │ N/A │ N/A │ N/A │ +│ Request throughput (per sec) │ 2.87 │ N/A │ N/A │ N/A │ N/A │ N/A │ +└───────────────────────────────────┴────────┴────────┴────────┴────────┴────────┴────────┘ +2024-10-14 22:44 [INFO] genai_perf.export_data.json_exporter:62 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.json +2024-10-14 22:44 [INFO] genai_perf.export_data.csv_exporter:71 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.csv +``` + +
+ +6. Use the OpenAI python client directly: +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:9000/v1", + api_key="EMPTY", +) + +model = "llama-3.1-8b-instruct" +completion = client.chat.completions.create( + model=model, + messages=[ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "What are LLMs?"}, + ], + max_tokens=256, +) + +print(completion.choices[0].message.content) +``` + +7. Run tests (NOTE: The server should not be running, the tests will handle starting/stopping the server as necessary): +```bash +cd /opt/tritonserver/python/openai/ +pip install -r requirements-test.txt + +pytest -v tests/ +``` + +## TensorRT-LLM + +0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: + - [Triton CLI](https://github.com/triton-inference-server/triton_cli/) + - [TRT-LLM Backend Quickstart](https://github.com/triton-inference-server/tensorrtllm_backend?tab=readme-ov-file#quick-start) + +1. Launch the container: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + -e TRTLLM_ORCHESTRATOR=1 \ + nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 +``` + +2. Install dependencies inside the container: +```bash +# Install python bindings for tritonserver and tritonfrontend +pip install /opt/tritonserver/python/triton*.whl + +# Install application requirements +git clone https://github.com/triton-inference-server/server.git +cd server/python/openai/ +pip install -r requirements.txt +``` + +2. Launch the OpenAI server: +```bash +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository path/to/models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +# MODEL should be the client-facing model name in your model repository for a pipeline like TRT-LLM. +# For example, this could also be "ensemble", or something like "gpt2" if generated from Triton CLI +MODEL="tensorrt_llm_bls" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-704c758c-8a84-11ef-b106-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "It looks like you're testing the system!", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1728948689, + "model": "llama-3-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +The other examples should be the same as vLLM, except that you should set `MODEL="tensorrt_llm_bls"` or `MODEL="ensemble"`, +everywhere applicable as seen in the example request above. + +## KServe Frontends + +To support serving requests through both the OpenAI-Compatible and +KServe Predict v2 frontends to the same running Triton Inference Server, +the `tritonfrontend` python bindings are included for optional use in this +application as well. + +You can opt-in to including these additional frontends, assuming `tritonfrontend` +is installed, with `--enable-kserve-frontends` like below: + +``` +python3 openai_frontend/main.py \ + --model-repository tests/vllm_models \ + --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ + --enable-kserve-frontends +``` + +See `python3 openai_frontend/main.py --help` for more information on the +available arguments and default values. + +For more information on the `tritonfrontend` python bindings, see the docs +[here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md). + +## Model Parallelism Support + +- [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend)) + - ex: Configure `tensor_parallel_size: 2` in the + [model.json](https://github.com/triton-inference-server/vllm_backend/blob/main/samples/model_repository/vllm_model/1/model.json) +- [x] TensorRT-LLM ([Orchestrator Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#orchestrator-mode)) + - Set the following environment variable: `export TRTLLM_ORCHESTRATOR=1` +- [ ] TensorRT-LLM ([Leader Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#leader-mode)) + - Not currently supported diff --git a/docs/examples/model_repository/simple_identity/1/model.py b/docs/examples/model_repository/simple_identity/1/model.py index 906c173892..3e0db73aaa 100644 --- a/docs/examples/model_repository/simple_identity/1/model.py +++ b/docs/examples/model_repository/simple_identity/1/model.py @@ -1,4 +1,8 @@ +<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py # Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +======== +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +>>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,6 +28,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py import json import triton_python_backend_utils as pb_utils @@ -44,3 +49,24 @@ def execute(self, requests): out_tensor_0 = pb_utils.Tensor("OUTPUT0", in_0.as_numpy()) responses.append(pb_utils.InferenceResponse([out_tensor_0])) return responses +======== +name: pre-commit + +on: + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5.0.0 + with: + fetch-depth: 2 + - name: Get modified files + id: modified-files + run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT + - uses: actions/setup-python@v6.0.0 + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --files ${{ steps.modified-files.outputs.modified_files }} +>>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8824e2ed6a..9445464ebc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -90,6 +90,19 @@ endif() # find_package(re2 REQUIRED) +option(TRITON_ENABLE_MYSQL_ODBC "Build tritonserver with MySQL ODBC connection pool (requires ODBC::ODBC / unixODBC)" OFF) +if(TRITON_ENABLE_MYSQL_ODBC) + if(UNIX AND NOT APPLE AND NOT WIN32) + message( + STATUS + "TRITON_ENABLE_MYSQL_ODBC: ensure unixODBC development packages are installed " + "(e.g. Debian/Ubuntu: unixodbc-dev; RHEL: unixODBC-devel) so CMake can find ODBC." + ) + endif() + find_package(ODBC REQUIRED) + message(STATUS "Using ODBC ${ODBC_VERSION}") +endif() + # # tritonserver executable # @@ -101,8 +114,10 @@ add_executable( main.cc shared_memory_manager.cc triton_signal.cc + database_config.cc classification.h common.h + database_config.h shared_memory_manager.h triton_signal.h ) @@ -155,6 +170,17 @@ else() ) endif() +if(TRITON_ENABLE_MYSQL_ODBC) + target_sources( + main + PRIVATE + mysql_odbc_connection_pool.cc + mysql_odbc_connection_pool.h + ) + target_link_libraries(main PRIVATE ODBC::ODBC) + target_compile_definitions(main PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) +endif() + set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") @@ -180,6 +206,7 @@ target_link_libraries( triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-common-logging # from repo-common + triton-common-json # from repo-common (RapidJSON) triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) @@ -337,11 +364,13 @@ if(${TRITON_ENABLE_HTTP} list(APPEND HTTP_ENDPOINT_SRCS http_server.cc + multi_infer.cc orca_http.cc ) list(APPEND HTTP_ENDPOINT_HDRS http_server.h + http_server_macros.h orca_http.h ) @@ -739,6 +768,61 @@ if (NOT WIN32) RUNTIME DESTINATION bin ) + # + # transform (JSON -> RapidJSON, TRITONSERVER_Error reporting) + # + add_library(transform STATIC transform.cc transform.h) + + # Required when transform is linked into shared objects (e.g. py-bindings -> + # http-endpoint-library -> transform); otherwise relocations fail at link time. + set_target_properties( + transform + PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) + + target_compile_features(transform PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options( + transform + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + transform + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + target_link_libraries( + transform + PRIVATE + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-common-json # RapidJSON from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + ) + + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions(transform PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) + target_link_libraries(transform PRIVATE ODBC::ODBC) + endif() + + if((${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR + ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) AND + TARGET http-endpoint-library AND TARGET transform) + target_link_libraries(http-endpoint-library PRIVATE transform) + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions( + http-endpoint-library + PRIVATE TRITON_ENABLE_MYSQL_ODBC=1 + ) + endif() + endif() + if(${TRITON_ENABLE_GPU}) # # memory_alloc example diff --git a/src/database_config.cc b/src/database_config.cc new file mode 100644 index 0000000000..cb312dec5b --- /dev/null +++ b/src/database_config.cc @@ -0,0 +1,204 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "database_config.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::optional ReadEntireFile(const std::string& path, std::string* contents) +{ + std::ifstream in(path, std::ios::binary); + if (!in) { + return std::string("failed to open file: ") + path; + } + std::ostringstream ss; + ss << in.rdbuf(); + if (!in && !in.eof()) { + return std::string("failed to read file: ") + path; + } + *contents = ss.str(); + return std::nullopt; +} + +std::optional ExpectString(const rapidjson::Value& obj, const char* key, std::string* field, bool required) +{ + if (!obj.HasMember(key)) { + if (required) { + return std::string("missing required JSON field: ") + key; + } + field->clear(); + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsString()) { + return std::string("JSON field '") + key + "' must be a string"; + } + *field = v.GetString(); + return std::nullopt; +} + +std::optional OptionalInt(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsInt()) { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = v.GetInt(); + return std::nullopt; +} + +std::optional OptionalIntFlexible(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + int n = 0; + if (v.IsInt()) { + n = v.GetInt(); + } else if (v.IsUint()) { + if (v.GetUint() > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + const int64_t v64 = v.GetInt64(); + if (v64 < INT_MIN || v64 > INT_MAX) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else if (v.IsUint64()) { + const uint64_t v64 = v.GetUint64(); + if (v64 > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = n; + return std::nullopt; +} + +std::optional OptionalNonNegativeSize( + const rapidjson::Value& obj, const char* key, std::size_t* out, + std::size_t def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (v.IsUint64()) { + *out = static_cast(v.GetUint64()); + return std::nullopt; + } + if (v.IsInt64()) { + const int64_t n = v.GetInt64(); + if (n < 0) { + return std::string("JSON field '") + key + "' must be non-negative"; + } + *out = static_cast(n); + return std::nullopt; + } + return std::string("JSON field '") + key + "' must be an integer"; +} + +} // namespace + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out) +{ + std::string raw; + if (auto e = ReadEntireFile(path, &raw)) { + return e; + } + + rapidjson::Document doc; + doc.Parse(raw.c_str()); + if (doc.HasParseError()) { + return std::string("JSON parse error: ") +rapidjson::GetParseError_En(doc.GetParseError()) + " at offset " + std::to_string(doc.GetErrorOffset()); + } + if (!doc.IsObject()) { + return std::string("root JSON value must be an object"); + } + + DatabaseConfig c; + if (auto e = ExpectString(doc, "databaseIp", &c.database_ip, false)) return e; + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.front()))) c.database_ip.erase(0, 1); + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.back()))) c.database_ip.pop_back(); + + if (auto e = OptionalIntFlexible(doc, "databasePort", &c.database_port, 3306)) return e; + + if (c.database_port < 1 || c.database_port > 65535) return std::string("databasePort must be between 1 and 65535"); + + if (auto e = ExpectString(doc, "odbcDriverName", &c.odbc_driver_name, false)) return e; + + if (auto e = ExpectString(doc, "primaryDSNName", &c.primary_dsn_name, true)) return e; + + if (auto e = ExpectString(doc, "secondaryDSNName", &c.secondary_dsn_name, false)) return e; + + if (auto e = ExpectString(doc, "dsnUserName", &c.dsn_user_name, true)) return e; + + if (auto e = ExpectString(doc, "dsnUserPassword", &c.dsn_user_password, true)) return e; + + int dc_id_int = 0; + if (auto e = OptionalIntFlexible(doc, "dcId", &dc_id_int, 0)) return e; + if (dc_id_int < INT32_MIN || dc_id_int > INT32_MAX) return std::string("dcId is out of range for int32_t"); + c.dc_id = static_cast(dc_id_int); + + if (auto e = OptionalInt(doc, "queryRetryCount", &c.query_retry_count, 3)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "minPoolConnections", &c.min_pool_connections, 2)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "maxPoolConnections", &c.max_pool_connections, 5)) return e; + + if (c.min_pool_connections < 1) return std::string("minPoolConnections must be at least 1"); + + if (c.max_pool_connections < c.min_pool_connections) return std::string("maxPoolConnections must be greater than or equal to minPoolConnections"); + *out = std::move(c); + return std::nullopt; +} + +}} // namespace triton::server + diff --git a/src/database_config.h b/src/database_config.h new file mode 100644 index 0000000000..c1083ed020 --- /dev/null +++ b/src/database_config.h @@ -0,0 +1,56 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +namespace triton { namespace server { + +inline constexpr const char kTritonDmConfigJsonPath[] = "/etc/triton-dmconfig.json"; + +struct DatabaseConfig { + std::string database_ip; + int database_port{3306}; + std::string odbc_driver_name; + + std::string primary_dsn_name; + std::string secondary_dsn_name; + std::string dsn_user_name; + std::string dsn_user_password; + + int32_t dc_id{0}; + int query_retry_count{3}; + std::size_t min_pool_connections{2}; + std::size_t max_pool_connections{5}; +}; + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out); + +}} // namespace triton::server + diff --git a/src/http_server.cc b/src/http_server.cc index edec3aae0a..845caf0569 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -35,16 +35,19 @@ #include #include +#include +#include #include #include - +#include +#include "triton/common/triton_json.h" #include "classification.h" +#include "http_server_macros.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr -#include "triton/common/triton_json.h" namespace triton { namespace server { @@ -58,36 +61,6 @@ namespace triton { namespace server { } \ } while (false) -#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ - do { \ - TRITONSERVER_Error* err__ = (X); \ - if (err__ != nullptr) { \ - EVBufferAddErrorJson((REQ)->buffer_out, err__); \ - evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ - TRITONSERVER_ErrorDelete(err__); \ - return; \ - } \ - } while (false) - -#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ - do { \ - EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ - evhtp_send_reply((REQ), CODE); \ - return; \ - } while (false) - -#define RETURN_AND_RESPOND_IF_RESTRICTED( \ - REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ - do { \ - auto const& is_restricted_api = \ - RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ - auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ - if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ - return; \ - } \ - } while (false) - - namespace { int @@ -3226,6 +3199,27 @@ HTTPAPIServer::ForwardHeaders( return nullptr; // success } +TRITONSERVER_Error* +HTTPAPIServer::ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse* response, const uint32_t flags, + void* userp)) +{ + RETURN_IF_ERR(ForwardHeaders(req, irequest)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, InferRequestClass::InferRequestComplete, + request_release_payload)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator_, + reinterpret_cast(&infer_request->alloc_payload_), + infer_response_complete_fn, reinterpret_cast(infer_request))); + return TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); +} + void HTTPAPIServer::HandleGenerate( evhtp_request_t* req, const std::string& model_name, @@ -3694,25 +3688,11 @@ HTTPAPIServer::HandleInfer( request_id = ""; } - RETURN_AND_CALLBACK_IF_ERR(ForwardHeaders(req, irequest), error_callback); - auto request_release_payload = std::make_unique( irequest_shared, decompressed_buffer); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetReleaseCallback( - irequest, InferRequestClass::InferRequestComplete, - request_release_payload.get()), - error_callback); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator_, - reinterpret_cast(&infer_request->alloc_payload_), - InferRequestClass::InferResponseComplete, - reinterpret_cast(infer_request.get())), - error_callback); - - auto err = - TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); + auto err = ScheduleInferAsync( + req, irequest, infer_request.get(), request_release_payload.get(), + triton_trace); #ifdef TRITON_ENABLE_TRACING // Ownership of trace passed to Triton core, set trace to null to mark it // as no longer owned here. @@ -3775,17 +3755,23 @@ HTTPAPIServer::InferRequestClass::InferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request, bool register_fini_cancel_hook) : server_(server), req_(req), response_compression_type_(response_compression_type), response_count_(0), + register_fini_cancel_hook_(register_fini_cancel_hook), triton_request_(triton_request), shm_manager_(shm_manager) { evhtp_connection_t* htpconn = evhtp_request_get_connection(req); thread_ = htpconn->thread; - evhtp_request_pause(req); - evhtp_request_set_hook( - req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, - reinterpret_cast(this)); + if (pause_http_request) { + evhtp_request_pause(req); + } + if (register_fini_cancel_hook_) { + evhtp_request_set_hook( + req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, + reinterpret_cast(this)); + } } void @@ -3832,7 +3818,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( std::to_string(infer_request->response_count_)) .c_str()); } else if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( @@ -3862,7 +3848,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -3959,6 +3945,16 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( // Handle data. SHM outputs will not have an info. auto info = reinterpret_cast(userp); + if (json_only_out != nullptr) { + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "multi_infer sub-request: only plain JSON outputs are supported (no " + "shared memory, binary tensor data, or classification)"); + } + } + size_t element_count = 1; uint32_t batch_size = 0; @@ -4096,34 +4092,45 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( } evbuffer* response_body = response_placeholder; - switch (response_compression_type_) { - case DataCompressor::Type::DEFLATE: - case DataCompressor::Type::GZIP: { - auto compressed_buffer = evbuffer_new(); - auto err = DataCompressor::CompressData( - response_compression_type_, response_placeholder, compressed_buffer); - if (err == nullptr) { - response_body = compressed_buffer; - evbuffer_free(response_placeholder); - } else { - // just log the compression error and return the uncompressed data - LOG_VERBOSE(1) << "unable to compress response: " - << TRITONSERVER_ErrorMessage(err); - TRITONSERVER_ErrorDelete(err); - evbuffer_free(compressed_buffer); - response_compression_type_ = DataCompressor::Type::IDENTITY; + if (json_only_out == nullptr) { + switch (response_compression_type_) { + case DataCompressor::Type::DEFLATE: + case DataCompressor::Type::GZIP: { + auto compressed_buffer = evbuffer_new(); + auto err = DataCompressor::CompressData( + response_compression_type_, response_placeholder, + compressed_buffer); + if (err == nullptr) { + response_body = compressed_buffer; + evbuffer_free(response_placeholder); + } else { + // just log the compression error and return the uncompressed data + LOG_VERBOSE(1) << "unable to compress response: " + << TRITONSERVER_ErrorMessage(err); + TRITONSERVER_ErrorDelete(err); + evbuffer_free(compressed_buffer); + response_compression_type_ = DataCompressor::Type::IDENTITY; + } + break; } - break; + case DataCompressor::Type::IDENTITY: + case DataCompressor::Type::UNKNOWN: + // Do nothing for other cases + break; } - case DataCompressor::Type::IDENTITY: - case DataCompressor::Type::UNKNOWN: - // Do nothing for other cases - break; + SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); + evbuffer_add_buffer(req_->buffer_out, response_body); + } else { + if (!ordered_buffers.empty()) { + evbuffer_free(response_placeholder); + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "multi_infer sub-request: binary outputs are not supported"); + } + evbuffer_add_buffer(json_only_out, response_body); } - SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); - evbuffer_add_buffer(req_->buffer_out, response_body); // Destroy the evbuffer object as the data has been moved - // to HTTP response buffer + // to HTTP response buffer (or json_only_out) evbuffer_free(response_body); return nullptr; // success @@ -4194,7 +4201,7 @@ HTTPAPIServer::GenerateRequestClass::InferResponseComplete( TRITONSERVER_Error* err = nullptr; if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); } if (err != nullptr) { infer_request->AddErrorJson(err); @@ -4354,8 +4361,13 @@ HTTPAPIServer::GenerateRequestClass::SendChunkResponse(bool end) TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { + if (json_only_out != nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "JSON-only response aggregation is not supported for generate"); + } triton_response_ = response; RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -4635,12 +4647,15 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingOutput( return nullptr; // success } -void -HTTPAPIServer::Handle(evhtp_request_t* req) -{ - LOG_VERBOSE(1) << "HTTP request: " << req->method << " " - << req->uri->path->full; +void HTTPAPIServer::Handle(evhtp_request_t* req) { + LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (std::string(req->uri->path->full) == "/v2/multi_infer") { + HandleMultiInfer(req); + return; + } +#endif if (std::string(req->uri->path->full) == "/v2/models/stats") { // model statistics HandleModelStats(req); diff --git a/src/http_server.h b/src/http_server.h index 785c408041..40e3a65263 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -278,11 +278,13 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager); + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true); virtual ~InferRequestClass() { - if (req_ != nullptr) { + if (req_ != nullptr && register_fini_cancel_hook_) { evhtp_request_unset_hook(req_, evhtp_hook_on_request_fini); } req_ = nullptr; @@ -319,8 +321,11 @@ class HTTPAPIServer : public HTTPServer { static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); + // When json_only_out is non-null, write infer response JSON there only + // (no HTTP headers); used by POST /v2/multi_infer aggregation. virtual TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response); + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr); // Helper function to set infer response header in the form specified by // the endpoint protocol @@ -329,6 +334,8 @@ class HTTPAPIServer : public HTTPServer { uint32_t IncrementResponseCount(); + uint32_t ResponseCount() const { return response_count_.load(); } + // Only used if tracing enabled std::shared_ptr trace_; @@ -359,6 +366,8 @@ class HTTPAPIServer : public HTTPServer { // Counter to keep track of number of responses generated. std::atomic response_count_{0}; + const bool register_fini_cancel_hook_; + // Event hook for called before request deletion static evhtp_res RequestFiniHook(evhtp_request* req, void* arg); @@ -412,7 +421,8 @@ class HTTPAPIServer : public HTTPServer { // Response preparation TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response) override; + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr) override; void AddErrorJson(TRITONSERVER_Error* error); static void StartResponse(evthr_t* thr, void* arg, void* shared); @@ -506,11 +516,13 @@ class HTTPAPIServer : public HTTPServer { // [FIXME] extract to "infer" class virtual std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) { return std::unique_ptr(new InferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } // Helper function to retrieve infer request header in the form specified by @@ -541,6 +553,17 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Error* ForwardHeaders( evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest); + // ForwardHeaders, release/response callbacks, and ServerInferAsync (shared + // by HandleInfer and multi_infer sub-requests). + TRITONSERVER_Error* ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse*, const uint32_t, void*) = + InferRequestClass::InferResponseComplete); + static TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, @@ -573,6 +596,8 @@ class HTTPAPIServer : public HTTPServer { void HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); + // POST /v2/multi_infer — parallel infer for multiple models in one HTTP call. + void HandleMultiInfer(evhtp_request_t* req); void HandleModelStats( evhtp_request_t* req, const std::string& model_name = "", const std::string& model_version_str = ""); diff --git a/src/http_server_macros.h b/src/http_server_macros.h new file mode 100644 index 0000000000..3cd47a1b14 --- /dev/null +++ b/src/http_server_macros.h @@ -0,0 +1,41 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Shared preprocessor macros for HTTP API handlers. Macros are not C++ symbols +// and are not "in" a namespace; this header is included from within +// triton::server for consistency with http_server.cc. +// +// Prerequisites at expansion sites: HttpCodeFromError, EVBufferAddErrorJson, +// and (for RETURN_AND_RESPOND_IF_RESTRICTED) RespondIfRestricted must be +// visible — typically from the same translation unit's anonymous namespace and +// HTTPAPIServer member functions respectively. + +#pragma once + +#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ + do { \ + TRITONSERVER_Error* err__ = (X); \ + if (err__ != nullptr) { \ + EVBufferAddErrorJson((REQ)->buffer_out, err__); \ + evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ + TRITONSERVER_ErrorDelete(err__); \ + return; \ + } \ + } while (false) + +#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ + do { \ + EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ + evhtp_send_reply((REQ), CODE); \ + return; \ + } while (false) + +#define RETURN_AND_RESPOND_IF_RESTRICTED( \ + REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ + do { \ + auto const& is_restricted_api = \ + RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ + auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ + if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ + return; \ + } \ + } while (false) diff --git a/src/main.cc b/src/main.cc index b2eee17c68..38b21ec648 100644 --- a/src/main.cc +++ b/src/main.cc @@ -42,11 +42,20 @@ #include #include +#include +#include +#include #include #include +#include +#include #include #include +#include "database_config.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "mysql_odbc_connection_pool.h" +#endif // TRITON_ENABLE_MYSQL_ODBC #include "triton_signal.h" #ifdef TRITON_ENABLE_ASAN @@ -103,6 +112,113 @@ std::unique_ptr g_vertex_ai_service; triton::server::TritonServerParameters g_triton_params; +// Populated at startup when /etc/triton-dmconfig.json is present. +std::optional g_triton_dm_database_config; + +#ifdef TRITON_ENABLE_MYSQL_ODBC +// ODBC pool opened after successful config load (same lifetime as the process). +std::unique_ptr g_triton_dm_odbc_pool; + +// Registered with std::atexit after the pool is initialized so connections are +// released on normal return, exit(), and FAIL_IF_ERR paths. +extern "C" void TritonDmOdbcPoolAtExit(void) +{ + triton::server::SetGlobalMysqlOdbcPool(nullptr); + g_triton_dm_odbc_pool.reset(); +} + +void TritonModelsRefreshThreadMain() +{ + using std::chrono_literals::operator""min; + while (!triton::server::signal_exiting_) { + if (auto err = triton::server::UpdateTritonModelsData()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Triton ML models DB refresh failed: " << *err; +#endif // TRITON_ENABLE_LOGGING + } + std::unique_lock lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.wait_for(lock, 15min, [] { return triton::server::signal_exiting_; }); + } +} + +std::optional g_triton_models_refresh_thread; + +void StartTritonModelsRefreshThread() +{ + if (!g_triton_dm_odbc_pool) { + return; + } + g_triton_models_refresh_thread.emplace(TritonModelsRefreshThreadMain); +} + +void JoinTritonModelsRefreshThread() +{ + if (!g_triton_models_refresh_thread.has_value()) { + return; + } + if (g_triton_models_refresh_thread->joinable()) { + { + std::lock_guard lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.notify_all(); + } + g_triton_models_refresh_thread->join(); + } + g_triton_models_refresh_thread.reset(); +} +#endif // TRITON_ENABLE_MYSQL_ODBC + +void LoadTritonDmDatabaseConfigAtStartup() +{ + namespace fs = std::filesystem; + const std::string path(triton::server::kTritonDmConfigJsonPath); + if (!fs::exists(path)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Database config file '" << path << "' not found; continuing without DM database metadata."; +#else + std::cerr << "warning: database config file '" << path << "' not found; continuing without DM database metadata." << std::endl; +#endif // TRITON_ENABLE_LOGGING + return; + } + + triton::server::DatabaseConfig cfg; + if (auto err = triton::server::LoadDatabaseConfigFromJsonFile(path, &cfg)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to load '" << path << "': " << *err; +#else + std::cerr << "Failed to load '" << path << "': " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + exit(1); + } + + g_triton_dm_database_config = std::move(cfg); + +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "Loaded database config from '" << path << "' (databaseIp='" << g_triton_dm_database_config->database_ip << "', databasePort=" << g_triton_dm_database_config->database_port << ")"; +#endif // TRITON_ENABLE_LOGGING + +#ifdef TRITON_ENABLE_MYSQL_ODBC + g_triton_dm_odbc_pool = std::make_unique(*g_triton_dm_database_config); + if (auto err = g_triton_dm_odbc_pool->Initialize()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to initialize MySQL ODBC connection pool: " << *err; +#else + std::cerr << "Failed to initialize MySQL ODBC connection pool: " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + g_triton_dm_odbc_pool.reset(); + exit(1); + } + if (std::atexit(TritonDmOdbcPoolAtExit) != 0) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "std::atexit failed; ODBC pool will still be destroyed at static exit"; +#endif // TRITON_ENABLE_LOGGING + } + triton::server::SetGlobalMysqlOdbcPool(g_triton_dm_odbc_pool.get()); +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "MySQL ODBC connection pool started ("<< g_triton_dm_database_config->max_pool_connections << " connections to DSN '" << g_triton_dm_database_config->primary_dsn_name << "')"; +#endif // TRITON_ENABLE_LOGGING +#endif // TRITON_ENABLE_MYSQL_ODBC +} + #ifdef TRITON_ENABLE_GRPC TRITONSERVER_Error* StartGrpcService( @@ -469,6 +585,8 @@ main(int argc, char** argv) LOG_SET_OUT_FILE(g_triton_params.log_file_); #endif // TRITON_ENABLE_LOGGING + LoadTritonDmDatabaseConfigAtStartup(); + // Trace manager. triton::server::TraceManager* trace_manager; @@ -501,6 +619,10 @@ main(int argc, char** argv) exit(1); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + StartTritonModelsRefreshThread(); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Wait until a signal terminates the server... while (!triton::server::signal_exiting_) { // If enabled, poll the model repository to see if there have been @@ -521,6 +643,10 @@ main(int argc, char** argv) triton::server::signal_exit_cv_.wait_for(lock, wait_timeout); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + JoinTritonModelsRefreshThread(); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Stop the HTTP[, gRPC, and metrics] endpoints, and update exit timeout. uint32_t exit_timeout_secs = g_triton_params.exit_timeout_secs_; StopEndpoints(&exit_timeout_secs); diff --git a/src/multi_infer.cc b/src/multi_infer.cc new file mode 100644 index 0000000000..e7659446e5 --- /dev/null +++ b/src/multi_infer.cc @@ -0,0 +1,635 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "http_server.h" + +#include "classification.h" +#include "common.h" +#include "transform.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +#include "http_server_macros.h" + +namespace { + +constexpr size_t kMaxMultiInferRequests = 16; + +int HttpCodeFromError(TRITONSERVER_Error* error) { + if (error == nullptr) { + return EVHTP_RES_OK; + } + switch (TRITONSERVER_ErrorCode(error)) { + case TRITONSERVER_ERROR_INTERNAL: + return EVHTP_RES_SERVERR; + case TRITONSERVER_ERROR_NOT_FOUND: + return EVHTP_RES_NOTFOUND; + case TRITONSERVER_ERROR_UNAVAILABLE: + return EVHTP_RES_SERVUNAVAIL; + case TRITONSERVER_ERROR_UNSUPPORTED: + return EVHTP_RES_NOTIMPL; + case TRITONSERVER_ERROR_UNKNOWN: + case TRITONSERVER_ERROR_INVALID_ARG: + case TRITONSERVER_ERROR_ALREADY_EXISTS: + case TRITONSERVER_ERROR_CANCELLED: + return EVHTP_RES_BADREQ; + } + + return EVHTP_RES_BADREQ; +} + +void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { + triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); + response.AddStringRef("error", message, strlen(message)); + + triton::common::TritonJson::WriteBuffer buffer_json; + response.Write(&buffer_json); + + evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); +} + +void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { + const char* message = TRITONSERVER_ErrorMessage(err); + EVBufferAddErrorJson(buffer, message); +} + +void AddContentTypeHeader(evhtp_request_t* req, const char* type) { + auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); + if (content_header) { + evhtp_header_rm_and_free(req->headers_out, content_header); + } + + evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); +} + +TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { + *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); + { + triton::common::TritonJson::Value v; + if (slot.Find("id", &v)) { + RETURN_IF_ERR(infer_json->Add("id", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (!slot.Find("inputs", &v)) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); + } + RETURN_IF_ERR(infer_json->Add("inputs", std::move(v))); + } + { + triton::common::TritonJson::Value v; + if (slot.Find("outputs", &v)) { + RETURN_IF_ERR(infer_json->Add("outputs", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (slot.Find("parameters", &v)) { + RETURN_IF_ERR(infer_json->Add("parameters", std::move(v))); + } + } + return nullptr; +} + +TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) +{ + ver_out->clear(); + triton::common::TritonJson::Value mv; + if (!slot.Find("model_version", &mv)) { + return nullptr; + } + if (mv.IsString()) { + const char* s; + size_t len; + RETURN_IF_ERR(mv.AsString(&s, &len)); + ver_out->assign(s, len); + return nullptr; + } + if (mv.IsNumber()) { + int64_t iv; + RETURN_IF_ERR(mv.AsInt(&iv)); + *ver_out = std::to_string(iv); + return nullptr; + } + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); +} + +class MultiInferAggregator : public std::enable_shared_from_this { + private: + struct FinishPayload { + std::shared_ptr agg; + }; + + public: + MultiInferAggregator(evhtp_request_t* req, size_t slot_count, evthr_t* reply_thread, std::vector> irequests) + : req_(req), n_(slot_count), reply_thread_(reply_thread), + irequests_(std::move(irequests)), success_json_(slot_count), + error_text_(slot_count), have_error_(slot_count, 0){} + + std::shared_ptr IrequestAt(size_t i) const + { + return irequests_[i]; + } + + void CancelAllSubRequests() + { + std::lock_guard lk(mu_); + if (cancel_sent_) { + return; + } + cancel_sent_ = true; + for (auto& ir : irequests_) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); + } + } + } + + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, const std::string& response_json) { + std::shared_ptr self; + { + std::lock_guard lk(mu_); + if (finalize_err != nullptr) { + have_error_[slot] = 1; + error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); + TRITONSERVER_ErrorDelete(finalize_err); + if (!cancel_sent_) { + cancel_sent_ = true; + for (auto& ir : irequests_) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); + } + } + } + } else { + success_json_[slot] = response_json; + } + done_count_++; + if (done_count_ < n_ || reply_scheduled_) { + return; + } + reply_scheduled_ = true; + self = shared_from_this(); + } + + auto* fp = new FinishPayload{std::move(self)}; + evthr_defer(reply_thread_, FinishThunk, fp); + } + + private: + static void FinishThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + std::unique_ptr fp(static_cast(arg)); + fp->agg->WriteHttpReply(); + } + + void WriteHttpReply() { + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value responses(root, triton::common::TritonJson::ValueType::ARRAY); + for (size_t i = 0; i < n_; ++i) { + if (have_error_[i]) { + triton::common::TritonJson::Value item(root, triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); + TRITONSERVER_Error* ae = err_part.AddString("message", error_text_[i]); + if (ae != nullptr) { + LOG_TRITONSERVER_ERROR(ae, "multi_infer: building error JSON"); + TRITONSERVER_ErrorDelete(ae); + } + TRITONSERVER_Error* be = item.Add("error", std::move(err_part)); + if (be != nullptr) { + LOG_TRITONSERVER_ERROR(be, "multi_infer: building error JSON"); + TRITONSERVER_ErrorDelete(be); + } + TRITONSERVER_Error* ce = responses.Append(std::move(item)); + if (ce != nullptr) { + LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); + TRITONSERVER_ErrorDelete(ce); + } + } + else { + triton::common::TritonJson::Value item; + TRITONSERVER_Error* perr = item.Parse(success_json_[i].c_str(), success_json_[i].size()); + if (perr != nullptr) { + triton::common::TritonJson::Value wrap(root, triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); + TRITONSERVER_Error* ae = err_part.AddString("message", TRITONSERVER_ErrorMessage(perr)); + TRITONSERVER_ErrorDelete(perr); + if (ae != nullptr) { + TRITONSERVER_ErrorDelete(ae); + } + TRITONSERVER_Error* be = wrap.Add("error", std::move(err_part)); + if (be != nullptr) { + TRITONSERVER_ErrorDelete(be); + } + TRITONSERVER_Error* ce = responses.Append(std::move(wrap)); + if (ce != nullptr) { + TRITONSERVER_ErrorDelete(ce); + } + } else { + TRITONSERVER_Error* ce = responses.Append(std::move(item)); + if (ce != nullptr) { + LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); + TRITONSERVER_ErrorDelete(ce); + } + } + } + } + + TRITONSERVER_Error* re = root.Add("responses", std::move(responses)); + if (re != nullptr) { + LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, re); + evhtp_send_reply(req_, HttpCodeFromError(re)); + TRITONSERVER_ErrorDelete(re); + evhtp_request_resume(req_); + return; + } + + triton::common::TritonJson::WriteBuffer wb; + TRITONSERVER_Error* we = root.Write(&wb); + if (we != nullptr) { + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, we); + evhtp_send_reply(req_, HttpCodeFromError(we)); + TRITONSERVER_ErrorDelete(we); + } + else { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); + evhtp_send_reply(req_, EVHTP_RES_OK); + } + evhtp_request_resume(req_); + } + + evhtp_request_t* req_; + const size_t n_; + evthr_t* reply_thread_; + std::vector> irequests_; + + std::mutex mu_; + size_t done_count_{0}; + std::vector success_json_; + std::vector error_text_; + std::vector have_error_; + bool cancel_sent_{false}; + bool reply_scheduled_{false}; +}; + +class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { + public: + MultiInferShardRequest(TRITONSERVER_Server* server, evhtp_request_t* req, + DataCompressor::Type response_compression_type, + const std::shared_ptr& triton_request, + const std::shared_ptr& shm_manager, + std::shared_ptr aggregator, const size_t slot) + : HTTPAPIServer::InferRequestClass(server, req, response_compression_type, triton_request, shm_manager, false /* pause */, false /* fini hook */), aggregator_(std::move(aggregator)), slot_(slot){} + + static void InferResponseComplete(TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { + auto* infer_request = reinterpret_cast(userp); + + if (response != nullptr) { + ++infer_request->response_count_; + } + + TRITONSERVER_Error* err = nullptr; + evbuffer* shard_json = evbuffer_new(); + if (infer_request->response_count_ != 1) { + const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); + } else if (response != nullptr) { + err = infer_request->FinalizeResponse(response, shard_json); +#ifdef TRITON_ENABLE_TRACING + if (infer_request->trace_ != nullptr) { + infer_request->trace_->CaptureTimestamp("INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); + } +#endif // TRITON_ENABLE_TRACING + } + + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); + + std::string json_fragment; + if (err == nullptr) { + const size_t len = evbuffer_get_length(shard_json); + if (len > 0) { + const unsigned char* p = evbuffer_pullup(shard_json, -1); + if (p != nullptr) { + json_fragment.assign(reinterpret_cast(p), len); + } + } + } + evbuffer_free(shard_json); + + if (err != nullptr) { + infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); + } + else { + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment); + } + + if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { + return; + } + evthr_defer(infer_request->thread_, DeleteMultiInferShardRequestThunk, infer_request); + } + + private: + static void DeleteMultiInferShardRequestThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + delete reinterpret_cast(arg); + } + + std::shared_ptr aggregator_; + const size_t slot_; +}; + +} // namespace + +void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + + if (req->method != htp_method_POST) { + RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); + } + + evhtp_request_pause(req); + + evbuffer* decompressed_buffer = nullptr; + TRITONSERVER_Error* derr = DecompressBuffer(req, &decompressed_buffer); + if (derr != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, derr); + evhtp_send_reply(req, HttpCodeFromError(derr)); + TRITONSERVER_ErrorDelete(derr); + evhtp_request_resume(req); + return; + } + + evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; + + triton::common::TritonJson::Value root; + TRITONSERVER_Error* err = nullptr; + const size_t body_len = evbuffer_get_length(body_buf); + std::vector body_copy(body_len); + if (body_len > 0) { + const ssize_t nread = evbuffer_copyout(body_buf, body_copy.data(), body_len); + if (nread < 0 || static_cast(nread) != body_len) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); + } + } + if (err == nullptr) { + const std::string body_json(body_copy.data(), body_len); + rapidjson::Document parsed; + err = triton::server::ParseRequest(body_json, server_.get(), &parsed); + if (err == nullptr) { + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + parsed.Accept(writer); + const std::string transformed(sb.GetString(), sb.GetSize()); + err = root.Parse(transformed.c_str(), transformed.size()); + } + } + if (decompressed_buffer != nullptr) { + evbuffer_free(decompressed_buffer); + decompressed_buffer = nullptr; + } + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + triton::common::TritonJson::Value requests; + if (!root.Find("requests", &requests)) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + const size_t n = requests.ArraySize(); + if (n == 0) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + if (n > kMaxMultiInferRequests) { + const std::string lim = "At most " + std::to_string(kMaxMultiInferRequests) + " sub-requests are allowed per multi_infer call"; + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, lim.c_str()); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + struct SlotPrep { + std::string model_name; + int64_t model_version{0}; + std::string infer_body_json; + }; + std::vector slots; + slots.reserve(n); + + for (size_t i = 0; i < n; ++i) { + triton::common::TritonJson::Value slot; + err = requests.At(i, &slot); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + const char* mn_c; + size_t mn_len; + err = slot.MemberAsString("model_name", &mn_c, &mn_len); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + SlotPrep prep; + prep.model_name.assign(mn_c, mn_len); + std::string ver_str; + err = GetModelVersionStringFromSlot(slot, &ver_str); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = GetModelVersionFromString(ver_str, &prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = CheckTransactionPolicy(req, prep.model_name, prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + triton::common::TritonJson::Value infer_only; + err = CopyInferSlotBodyJson(slot, &infer_only); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + triton::common::TritonJson::WriteBuffer wb; + err = infer_only.Write(&wb); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + prep.infer_body_json.assign(wb.Base(), wb.Size()); + slots.push_back(std::move(prep)); + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), slots[i].model_name.c_str(), slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused multi_infer sub-request"); + } + } + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r),"deleting HTTP multi_infer sub-request"); + }); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); + + for (size_t i = 0; i < n; ++i) { + evbuffer* body_i = evbuffer_new(); + evbuffer_add(body_i, slots[i].infer_body_json.data(), slots[i].infer_body_json.size()); + const int32_t content_length = static_cast(evbuffer_get_length(body_i)); + size_t header_length = 0; + err = GetInferenceHeaderLength(req, content_length, &header_length); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); + + err = EVRequestToTritonRequest(req, slots[i].model_name, irequests[i].get(), body_i, shard.get(), header_length); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + evbuffer_free(body_i); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto rel = std::make_unique(irequests[i], body_i); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } +} + +}} // namespace triton::server diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc new file mode 100644 index 0000000000..a563dc7629 --- /dev/null +++ b/src/mysql_odbc_connection_pool.cc @@ -0,0 +1,697 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "mysql_odbc_connection_pool.h" + +#ifdef TRITON_ENABLE_LOGGING +#include "triton/common/logging.h" +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::atomic g_global_mysql_odbc_pool{nullptr}; + +std::string BuildMySqlDriverConnectString(const DatabaseConfig& c) +{ + std::string driver = c.odbc_driver_name; + if (driver.empty()) { + driver = "MySQL ODBC 9.7 Unicode Driver"; + } else if (driver == "MySQL ODBC 8.0 Unicode Driver") { + driver = "MySQL ODBC 9.7 Unicode Driver"; + } + std::ostringstream conn; + conn << "DRIVER={" << driver << "};" << "SERVER=" << c.database_ip << ";" << "PORT=" << c.database_port << ";" << "UID={" << c.dsn_user_name << "};" << "PWD={" << c.dsn_user_password << "};"; + return conn.str(); +} + +} // namespace + +PooledOdbcConnection::PooledOdbcConnection() = default; +PooledOdbcConnection::PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc) : pool_(pool), dbc_(dbc){} +PooledOdbcConnection::~PooledOdbcConnection() +{ + Release(); +} + +PooledOdbcConnection::PooledOdbcConnection(PooledOdbcConnection&& other) noexcept : pool_(other.pool_), dbc_(other.dbc_) +{ + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; +} + +PooledOdbcConnection& PooledOdbcConnection::operator=(PooledOdbcConnection&& other) noexcept +{ + if (this != &other) { + Release(); + pool_ = other.pool_; + dbc_ = other.dbc_; + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; + } + return *this; +} + +void PooledOdbcConnection::Release() +{ + if (pool_ != nullptr && dbc_ != SQL_NULL_HDBC) { + pool_->ReturnConnection(dbc_); + } + pool_ = nullptr; + dbc_ = SQL_NULL_HDBC; +} + +MysqlOdbcConnectionPool::MysqlOdbcConnectionPool(DatabaseConfig config) : config_(std::move(config)){} +MysqlOdbcConnectionPool::~MysqlOdbcConnectionPool() +{ + std::lock_guard lk(mu_); + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } +} + +std::optional MysqlOdbcConnectionPool::Initialize() +{ + auto cleanup_partial = [this]() { + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } + }; + + SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &henv_); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLAllocHandle(ENV) failed"); + } + + rc = SQLSetEnvAttr(henv_, SQL_ATTR_ODBC_VERSION, (SQLPOINTER)SQL_OV_ODBC3, 0); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + return std::string("SQLSetEnvAttr failed"); + } + + const std::size_t pool_size = config_.max_pool_connections; + all_handles_.reserve(pool_size); + + for (std::size_t i = 0; i < pool_size; ++i) { + SQLHDBC dbc = SQL_NULL_HDBC; + rc = SQLAllocHandle(SQL_HANDLE_DBC, henv_, &dbc); + if (!SQL_SUCCEEDED(rc)) { + cleanup_partial(); + return std::string("SQLAllocHandle(DBC) failed"); + } + + if (!config_.database_ip.empty()) { + const std::string conn_str = BuildMySqlDriverConnectString(config_); + SQLCHAR out_conn[1024]{}; + SQLSMALLINT out_conn_len = 0; + rc = SQLDriverConnect(dbc, nullptr, reinterpret_cast(const_cast(conn_str.data())), + SQL_NTS, out_conn, sizeof(out_conn), &out_conn_len, SQL_DRIVER_NOPROMPT); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLDriverConnect failed"); + } + } else { + rc = SQLConnect(dbc,reinterpret_cast(const_cast(config_.primary_dsn_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_password.data())), + SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLConnect failed"); + } + } + + all_handles_.push_back(dbc); + free_.push_back(dbc); + } + + return std::nullopt; +} + +PooledOdbcConnection MysqlOdbcConnectionPool::Acquire() +{ + std::unique_lock lk(mu_); + cv_.wait(lk, [this] { return !free_.empty(); }); + SQLHDBC dbc = free_.front(); + free_.pop_front(); + return PooledOdbcConnection(this, dbc); +} + +void MysqlOdbcConnectionPool::ReturnConnection(SQLHDBC dbc) +{ + { + std::lock_guard lk(mu_); + free_.push_back(dbc); + } + cv_.notify_one(); +} + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool) +{ + g_global_mysql_odbc_pool.store(pool, std::memory_order_release); +} + +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool() +{ + return g_global_mysql_odbc_pool.load(std::memory_order_acquire); +} + +namespace { + + +constexpr const char kSqlBtModelsMaxTs[] = "SELECT UNIX_TIMESTAMP(MAX(update_timestamp)) FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1"; +constexpr const char kSqlBtModelsForDc[] = "SELECT campaign_id, LOWER(model_name), feature_mapping, feature_sequence, applicable_campaigns FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1 AND dc_id = ?"; +constexpr size_t kModelNameBuf = kMaxModelNameLen + 1; +constexpr size_t kFeatureMappingBuf = kMaxFeatureMappingJsonLen + 1; +constexpr size_t kFeatureSequenceBuf = kMaxFeatureSequenceLen + 1; +constexpr size_t kApplicableCampaignsBuf = kMaxApplicableCampaignsLen + 1; + +struct OdbcStmt { + SQLHSTMT h{SQL_NULL_HSTMT}; + explicit OdbcStmt(SQLHDBC dbc) + { + const SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_STMT, dbc, &h); + if (!SQL_SUCCEEDED(rc)) { + h = SQL_NULL_HSTMT; + } + } + ~OdbcStmt() + { + if (h != SQL_NULL_HSTMT) { + SQLFreeHandle(SQL_HANDLE_STMT, h); + } + } + OdbcStmt(const OdbcStmt&) = delete; + OdbcStmt& operator=(const OdbcStmt&) = delete; +}; + +void Trim(std::string* s) +{ + if (s == nullptr || s->empty()) { + return; + } + const auto not_space = [](unsigned char c) { return !std::isspace(c); }; + auto b = std::find_if(s->begin(), s->end(), not_space); + auto e = std::find_if(s->rbegin(), s->rend(), not_space).base(); + if (b >= e) { + s->clear(); + } else { + *s = std::string(b, e); + } +} + +std::string SqlCharBufferToString(const std::vector& buf, SQLLEN cb) +{ + if (cb == SQL_NULL_DATA) { + return {}; + } + const char* p = reinterpret_cast(buf.data()); + if (cb < 0) { + return std::string(p); + } + return std::string(p, static_cast(cb)); +} + +std::string JsonScalarToString(const rapidjson::Value& v) +{ + if (v.IsString()) { + return std::string(v.GetString(), v.GetStringLength()); + } + if (v.IsBool()) { + return v.GetBool() ? "true" : "false"; + } + if (v.IsInt()) { + return std::to_string(v.GetInt()); + } + if (v.IsUint()) { + return std::to_string(v.GetUint()); + } + if (v.IsInt64()) { + return std::to_string(v.GetInt64()); + } + if (v.IsUint64()) { + return std::to_string(v.GetUint64()); + } + if (v.IsDouble()) { + return std::to_string(v.GetDouble()); + } + if (v.IsNull()) { + return {}; + } + return {}; +} + +std::optional FetchMaxUnixTimestampFromDbc(SQLHDBC dbc, const char* sql, int64_t* out_ts) +{ + *out_ts = 0; + OdbcStmt st(dbc); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + SQLRETURN rc = SQLExecDirect(st.h, reinterpret_cast(const_cast(sql)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecDirect failed"); + } + int64_t ts = 0; + SQLLEN cb_ts = 0; + rc = SQLBindCol(st.h, 1, SQL_C_SBIGINT, &ts, 0, &cb_ts); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol failed"); + } + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + return std::nullopt; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_ts != SQL_NULL_DATA) { + *out_ts = ts; + } + return std::nullopt; +} + +} // namespace + +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out) +{ + out->clear(); + std::size_t start = 0; + while (start < s.size()) { + const std::size_t comma = s.find(',', start); + std::string piece = (comma == std::string::npos) ? s.substr(start) : s.substr(start, comma - start); + Trim(&piece); + if (!piece.empty()) { + out->push_back(std::move(piece)); + } + if (comma == std::string::npos) { + break; + } + start = comma + 1; + } +} + +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out) +{ + out->clear(); + std::string t = s; + Trim(&t); + if (t.empty()) { + return std::nullopt; + } + if (t.size() == 2 && (t[0] == 'n' || t[0] == 'N') && (t[1] == 'a' || t[1] == 'A')) { + return std::nullopt; + } + + std::vector tokens; + SplitCommaSeparatedStrings(t, &tokens); + if (tokens.empty()) { + return std::nullopt; + } + + for (const auto& tok : tokens) { + char* endptr = nullptr; + const long v = std::strtol(tok.c_str(), &endptr, 10); + if (endptr == tok.c_str() || *endptr != '\0') { + return std::string("invalid campaign id token: ") + tok; + } + out->push_back(static_cast(v)); + } + return std::nullopt; +} + +int GetFeatureMappingIdx(const char* feature_name, const char* feature, const FeatureMappingTables* feature_mapping) +{ + if (feature_mapping == nullptr || feature_name == nullptr || feature == nullptr) { + return -1; + } + const auto outer = feature_mapping->find(feature_name); + if (outer == feature_mapping->end()) { + return -1; + } + const auto inner = outer->second.value_to_index.find(feature); + if (inner == outer->second.value_to_index.end()) { + return -1; + } + return inner->second; +} + +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error) +{ + out->clear(); + if (parse_error != nullptr) { + parse_error->clear(); + } + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + if (parse_error != nullptr) { + *parse_error = std::string("JSON parse error at offset ") + std::to_string(doc.GetErrorOffset()) + ": " + rapidjson::GetParseError_En(doc.GetParseError()); + } + return false; + } + if (!doc.IsObject()) { + if (parse_error != nullptr) { + *parse_error = "feature_mapping root must be a JSON object"; + } + return false; + } + + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + if (!it->name.IsString()) { + continue; + } + const std::string feature_name(it->name.GetString(), it->name.GetStringLength()); + if (feature_name.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "feature name exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1"; + } + return false; + } + if (!it->value.IsArray()) { + if (parse_error != nullptr) { + *parse_error = "feature '" + feature_name + "' value is not a JSON array"; + } + return false; + } + const rapidjson::Value& arr = it->value; + FeatureValueIndexMap table; + table.values.reserve(arr.Size()); + for (rapidjson::SizeType i = 0; i < arr.Size(); ++i) { + const std::string cell = JsonScalarToString(arr[i]); + if (cell.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "categorical value exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1 for feature '" + feature_name + "'"; + } + return false; + } + table.values.push_back(cell); + table.value_to_index[cell] = static_cast(i); + } + (*out)[feature_name] = std::move(table); + } + return true; +} + +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts) +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + return FetchMaxUnixTimestampFromDbc(conn.handle(), kSqlBtModelsMaxTs, out_ts); +} + +namespace { + +bool TryMergeRowIntoCampaignMap(const LightgbmBtModelRow& row, CampaignToFeatureMappings& out) +{ + const CampaignBtModelBundle bundle{row.model_name, row.feature_mapping, row.feature_sequence}; + + if (row.campaign_id != 0) { + if (out.find(row.campaign_id) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << row.campaign_id << " already exists in triton models map; skipping row"; +#endif + return false; + } + out[row.campaign_id] = bundle; + return true; + } + + if (row.applicable_campaigns.empty()) { + if (out.find(0) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign 0 already exists in triton models map; skipping row"; +#endif + return false; + } + out[0] = bundle; + return true; + } + + bool any_inserted = false; + for (int32_t cid : row.applicable_campaigns) { + if (out.find(cid) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << cid << " already exists in triton models map"; +#endif + continue; + } + out[cid] = bundle; + any_inserted = true; + } + return any_inserted; +} + +} // namespace + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map) +{ + out_campaign_map.clear(); + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + const int32_t dc_id = pool->Config().dc_id; + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + + OdbcStmt st(conn.handle()); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + + SQLRETURN rc = SQLPrepare(st.h, reinterpret_cast(const_cast(kSqlBtModelsForDc)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLPrepare failed"); + } + + SQLINTEGER dc_param = static_cast(dc_id); + rc = SQLBindParameter(st.h, 1, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, &dc_param, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindParameter failed"); + } + + rc = SQLExecute(st.h); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecute failed"); + } + + SQLINTEGER s_campaign_id = 0; + SQLLEN cb_campaign_id = 0; + std::vector model_buf(kModelNameBuf + 1, 0); + SQLLEN cb_model_name = 0; + std::vector mapping_buf(kFeatureMappingBuf + 1, 0); + SQLLEN cb_mapping = 0; + std::vector sequence_buf(kFeatureSequenceBuf + 1, 0); + SQLLEN cb_sequence = 0; + std::vector campaigns_buf(kApplicableCampaignsBuf + 1, 0); + SQLLEN cb_campaigns = 0; + + int col = 1; + rc = SQLBindCol(st.h, col++, SQL_C_SLONG, &s_campaign_id, 0, &cb_campaign_id); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(campaign_id) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, model_buf.data(), static_cast(model_buf.size()), &cb_model_name); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(model_name) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, mapping_buf.data(), static_cast(mapping_buf.size()), &cb_mapping); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_mapping) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, sequence_buf.data(), static_cast(sequence_buf.size()), &cb_sequence); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_sequence) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, campaigns_buf.data(), static_cast(campaigns_buf.size()), &cb_campaigns); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(applicable_campaigns) failed"); + } + + while (true) { + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + break; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_campaign_id == SQL_NULL_DATA) { + continue; + } + + LightgbmBtModelRow row; + row.campaign_id = static_cast(s_campaign_id); + + if (cb_model_name != SQL_NULL_DATA) { + row.model_name = SqlCharBufferToString(model_buf, cb_model_name); + Trim(&row.model_name); + } + + std::string mapping_json; + if (cb_mapping != SQL_NULL_DATA) { + mapping_json = SqlCharBufferToString(mapping_buf, cb_mapping); + Trim(&mapping_json); + } + std::string parse_err; + if (!ParseFeatureMappingJson(mapping_json, &row.feature_mapping, &parse_err)) { + return std::string("feature_mapping JSON: ") + parse_err; + } + + std::string seq_str; + if (cb_sequence != SQL_NULL_DATA) { + seq_str = SqlCharBufferToString(sequence_buf, cb_sequence); + Trim(&seq_str); + } + SplitCommaSeparatedStrings(seq_str, &row.feature_sequence); + + std::string camp_str; + if (cb_campaigns != SQL_NULL_DATA) { + camp_str = SqlCharBufferToString(campaigns_buf, cb_campaigns); + Trim(&camp_str); + } + if (auto err = ParseApplicableCampaignIds(camp_str, &row.applicable_campaigns)) { + return err; + } + + if (!TryMergeRowIntoCampaignMap(row, out_campaign_map)) { + continue; + } + } + + return std::nullopt; +} + +namespace { + std::array g_triton_campaign_feature_mappings; + std::atomic g_triton_models_active{0}; + std::atomic g_triton_models_modification_time{0}; +} + +bool IsTritonModelsModified() +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return false; + } + + int retry_count = pool->Config().query_retry_count; + if (retry_count < 0) { + retry_count = 0; + } + + int64_t last_updated = 0; + std::optional err; + do { + err = FetchLightgbmBtModelsMaxUpdateUnixSeconds(&last_updated); + if (!err) { + break; + } + } while (err.has_value() && retry_count--); + + if (err.has_value()) { + return false; + } + + const uint32_t lu = static_cast(last_updated); + const uint32_t prev = g_triton_models_modification_time.load(std::memory_order_relaxed); + if (lu > prev) { + g_triton_models_modification_time.store(lu, std::memory_order_relaxed); + return true; + } + return false; +} + +std::optional UpdateTritonModelsData() +{ + if (!IsTritonModelsModified()) { + return std::nullopt; + } + + CampaignToFeatureMappings by_campaign; + if (auto err = FetchLightgbmBtModelsForDc(by_campaign)) { + return err; + } + + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + const int inactive = 1 - idx; + g_triton_campaign_feature_mappings[inactive] = std::move(by_campaign); + g_triton_models_active.store(inactive, std::memory_order_release); + g_triton_campaign_feature_mappings[idx].clear(); + return std::nullopt; +} + +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings() +{ + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + return &g_triton_campaign_feature_mappings[idx]; +} + +}} // namespace triton::server \ No newline at end of file diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h new file mode 100644 index 0000000000..771fcefbfb --- /dev/null +++ b/src/mysql_odbc_connection_pool.h @@ -0,0 +1,163 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Connection pool over the MySQL ODBC driver using the ODBC API (unixODBC / +// iODBC). Build with -DTRITON_ENABLE_MYSQL_ODBC=ON and install unixodbc-dev +// (Debian/Ubuntu) plus a configured MySQL ODBC DSN. + +#pragma once + +#include "database_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +#include +#include + +namespace triton { namespace server { + +inline constexpr std::size_t kMaxModelNameLen = 100; +inline constexpr std::size_t kMaxFeatureMappingJsonLen = 16 * (1 << 20); +inline constexpr std::size_t kMaxFeatureSequenceLen = 1000; +inline constexpr std::size_t kMaxApplicableCampaignsLen = 4096; +inline constexpr std::size_t kMaxLightgbmUpdateTimestampLen = 256; +inline constexpr std::size_t kTritonFeatureMappingBuffSize = 256; +inline constexpr std::size_t kTritonFeatureMappingMaxTokenLen = + kTritonFeatureMappingBuffSize - 1; + +class MysqlOdbcConnectionPool; + +class PooledOdbcConnection { + public: + PooledOdbcConnection(); + ~PooledOdbcConnection(); + + PooledOdbcConnection(PooledOdbcConnection&& other) noexcept; + PooledOdbcConnection& operator=(PooledOdbcConnection&& other) noexcept; + + PooledOdbcConnection(const PooledOdbcConnection&) = delete; + PooledOdbcConnection& operator=(const PooledOdbcConnection&) = delete; + + SQLHDBC handle() const { return dbc_; } + explicit operator bool() const { return dbc_ != SQL_NULL_HDBC; } + + private: + friend class MysqlOdbcConnectionPool; + PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc); + + void Release(); + + MysqlOdbcConnectionPool* pool_{nullptr}; + SQLHDBC dbc_{SQL_NULL_HDBC}; +}; + +class MysqlOdbcConnectionPool { + public: + explicit MysqlOdbcConnectionPool(DatabaseConfig config); + ~MysqlOdbcConnectionPool(); + + MysqlOdbcConnectionPool(const MysqlOdbcConnectionPool&) = delete; + MysqlOdbcConnectionPool& operator=(const MysqlOdbcConnectionPool&) = delete; + + std::optional Initialize(); + + PooledOdbcConnection Acquire(); + + const DatabaseConfig& Config() const { return config_; } + + private: + friend class PooledOdbcConnection; + void ReturnConnection(SQLHDBC dbc); + + DatabaseConfig config_; + SQLHENV henv_{SQL_NULL_HENV}; + std::vector all_handles_; + std::deque free_; + std::mutex mu_; + std::condition_variable cv_; +}; + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool); +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool(); + +struct FeatureValueIndexMap { + std::vector values; + std::unordered_map value_to_index; +}; + +using FeatureMappingTables = std::unordered_map; + +// Legacy get_feature_mapping_idx: look up categorical index for `feature` +// (token string) under column `feature_name`. Returns -1 if any map or key is +// missing, or if `feature_name` / `feature` / `feature_mapping` is null. +int GetFeatureMappingIdx( + const char* feature_name, const char* feature, + const FeatureMappingTables* feature_mapping); + +// Loaded from `lightgbm_bt_models` per campaign_id (after merge rules). +struct CampaignBtModelBundle { + std::string model_name; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; +}; + +using CampaignToFeatureMappings = std::unordered_map; + +struct LightgbmBtModelRow { + int32_t campaign_id{0}; + std::string model_name; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; + std::vector applicable_campaigns; +}; + +std::optional FetchLightgbmFeatureMappingMaxUpdateUnixSeconds(int64_t* out_ts); +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts); + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map); +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error); +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out_tokens); +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out_ids); +bool IsTritonModelsModified(); +std::optional UpdateTritonModelsData(); +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings(); + +}} // namespace triton::server + diff --git a/src/sagemaker_server.h b/src/sagemaker_server.h index dcd40e66ac..ffb94322df 100644 --- a/src/sagemaker_server.h +++ b/src/sagemaker_server.h @@ -53,10 +53,12 @@ class SagemakerAPIServer : public HTTPAPIServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) : InferRequestClass( server, req, response_compression_type, triton_request, - shm_manager) + shm_manager, pause_http_request, register_fini_cancel_hook) { } using InferRequestClass::InferResponseComplete; @@ -123,12 +125,13 @@ class SagemakerAPIServer : public HTTPAPIServer { std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) - override + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) override { return std::unique_ptr(new SagemakeInferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, diff --git a/src/transform.cc b/src/transform.cc new file mode 100644 index 0000000000..eb2f46e0f7 --- /dev/null +++ b/src/transform.cc @@ -0,0 +1,349 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include "mysql_odbc_connection_pool.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, const triton::server::FeatureMappingTables* tables) { + if (tables == nullptr) { + return -1; + } + if (v.IsString()) { + return triton::server::GetFeatureMappingIdx(feature_name, v.GetString(), tables); + } + char num_buf[48]; + int n = 0; + if (v.IsInt()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%d", v.GetInt()); + } else if (v.IsUint()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%u", v.GetUint()); + } else if (v.IsInt64()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%lld", static_cast(v.GetInt64())); + } else if (v.IsUint64()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%llu", static_cast(v.GetUint64())); + } else { + return -1; + } + if (n <= 0 || static_cast(n) >= sizeof(num_buf)) { + return -1; + } + return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); +} +} // namespace + +namespace triton { namespace server { + +TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { + if (out_doc == nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); + } + + rapidjson::Document doc; + doc.Parse(json.data(), json.size()); + if (doc.HasParseError()) { + const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); + } + + if (server != nullptr && doc.IsObject() && doc.HasMember("imps")) { + const rapidjson::Value& imps_member = doc["imps"]; + if (imps_member.IsArray()) { + return GenerateInputVectors(doc, server, out_doc); + } + } + + *out_doc = std::move(doc); + return nullptr; +} + +TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unordered_set* out) { + if (out == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output set pointer is null"); + } + out->clear(); + + TRITONSERVER_Message* message = nullptr; + TRITONSERVER_Error* err = TRITONSERVER_ServerModelIndex(server, TRITONSERVER_INDEX_FLAG_READY, &message); + if (err != nullptr) { + return err; + } + + const char* buffer = nullptr; + size_t byte_size = 0; + err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); + if (err != nullptr) { + TRITONSERVER_MessageDelete(message); + return err; + } + const std::string index_json(buffer, byte_size); + TRITONSERVER_MessageDelete(message); + + rapidjson::Document index_doc; + index_doc.Parse(index_json.data(), index_json.size()); + if (index_doc.HasParseError()) { + const std::string parse_err = std::string("failed to parse model index JSON: ") + rapidjson::GetParseError_En(index_doc.GetParseError()) + " at offset " + std::to_string(index_doc.GetErrorOffset()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, parse_err.c_str()); + } + if (!index_doc.IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "model index JSON root is not an array"); + } + + out->reserve(static_cast(index_doc.Size())); + for (rapidjson::SizeType i = 0; i < index_doc.Size(); ++i) { + const rapidjson::Value& o = index_doc[i]; + if (!o.IsObject() || !o.HasMember("name")) { + continue; + } + const rapidjson::Value& n = o["name"]; + if (!n.IsString()) { + continue; + } + out->emplace(n.GetString()); + } + + return nullptr; +} + +TRITONSERVER_Error* AppendRowToNamedDoubleBuffers(NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, + const std::string& vector_name, const std::vector& row, size_t feature_count) { + if (buffers == nullptr || feature_counts == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer pointers are null"); + } + if (row.size() != feature_count || feature_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "row width does not match feature_count"); + } + + auto fc_it = feature_counts->find(vector_name); + if (fc_it == feature_counts->end()) { + (*feature_counts)[vector_name] = feature_count; + } else if (fc_it->second != feature_count) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); + } + + std::vector& buf = (*buffers)[vector_name]; + buf.insert(buf.end(), row.begin(), row.end()); + return nullptr; +} + +TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buffers, const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc) { + if (out_doc == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); + } + + std::vector names; + names.reserve(buffers.size()); + for (const auto& kv : buffers) { + names.push_back(kv.first); + } + std::sort(names.begin(), names.end()); + + rapidjson::Document doc(rapidjson::kObjectType); + auto& alloc = doc.GetAllocator(); + rapidjson::Value requests(rapidjson::kArrayType); + + for (const std::string& model_name : names) { + auto bc_it = buffers.find(model_name); + auto fc_it = feature_counts.find(model_name); + if (bc_it == buffers.end() || fc_it == feature_counts.end()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "internal buffer map inconsistency"); + } + const size_t feature_count = fc_it->second; + if (feature_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "zero feature_count for model"); + } + const std::vector& flat = bc_it->second; + if (flat.size() % feature_count != 0 || flat.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); + } + const size_t rows = flat.size() / feature_count; + + rapidjson::Value req(rapidjson::kObjectType); + req.AddMember("model_name", rapidjson::Value(model_name.c_str(), static_cast(model_name.size()), alloc).Move(), alloc); + + rapidjson::Value data(rapidjson::kArrayType); + data.Reserve(static_cast(flat.size()), alloc); + for (double v : flat) { + data.PushBack(v, alloc); + } + + rapidjson::Value shape(rapidjson::kArrayType); + shape.PushBack(static_cast(rows), alloc); + shape.PushBack(static_cast(feature_count), alloc); + + rapidjson::Value input0(rapidjson::kObjectType); + input0.AddMember("name", "input__0", alloc); + input0.AddMember("datatype", "FP32", alloc); + input0.AddMember("shape", shape, alloc); + input0.AddMember("data", data, alloc); + + rapidjson::Value inputs(rapidjson::kArrayType); + inputs.PushBack(input0, alloc); + + rapidjson::Value outputs(rapidjson::kArrayType); + rapidjson::Value out0(rapidjson::kObjectType); + out0.AddMember("name", "output__0", alloc); + outputs.PushBack(out0, alloc); + + req.AddMember("inputs", inputs, alloc); + req.AddMember("outputs", outputs, alloc); + requests.PushBack(req, alloc); + } + + doc.AddMember("requests", requests, alloc); + *out_doc = std::move(doc); + return nullptr; +} + +TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { + if (out_doc == nullptr || server == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); + } + if (!doc.IsObject() || !doc.HasMember("imps") || !doc["imps"].IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array"); + } + + const CampaignToFeatureMappings* cmap = ActiveCampaignToFeatureMappings(); + if (cmap == nullptr || cmap->empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); + } + + std::unordered_set ready_model_names; + TRITONSERVER_Error* err = GetReadyModelNames(server, &ready_model_names); + if (err != nullptr) { + return err; + } + if (ready_model_names.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "no ready models reported by server"); + } + + NamedDoubleBuffers buffers; + ModelNameToFeatureCount counts; + + const rapidjson::Value& imps = doc["imps"]; + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { + const rapidjson::Value& imp = imps[ii]; + if (!imp.IsObject()) { + continue; + } + if (!imp.HasMember("camps") || !imp["camps"].IsArray() || imp["camps"].Size() == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each impression must include a non-empty camps array"); + } + const rapidjson::Value& camps = imp["camps"]; + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { + const rapidjson::Value& camp = camps[ci]; + if (!camp.IsObject() || !camp.HasMember("cid") || !camp["cid"].IsInt()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each camp must be an object with integer 'cid'"); + } + const int32_t campaign_id = camp["cid"].GetInt(); + + auto cmap_it = cmap->find(campaign_id); + if(cmap_it == cmap->end()) { + cmap_it = cmap->find(0); + } + if (cmap_it == cmap->end()) { + const std::string unknown_campaign = std::string("unknown campaign_id ") + std::to_string(campaign_id); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, unknown_campaign.c_str()); + } + const std::string& model_name = cmap_it->second.model_name; + const std::vector& feature_sequence = cmap_it->second.feature_sequence; + const FeatureMappingTables& tables = cmap_it->second.feature_mapping; + + if (ready_model_names.find(model_name) == ready_model_names.end()) { + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + } + + std::vector row(feature_sequence.size(), 0.0); + int adsize_idx = -1; + + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + const char* fkey = feature.c_str(); + + if (feature == "adsize") { + adsize_idx = static_cast(fi); + continue; + } + + const rapidjson::Value* src = nullptr; + if (feature == "cookie" || feature == "rnk") { + if (camp.HasMember(fkey)) { + src = &camp[fkey]; + } + } + else if(feature == "campid") { + if (camp.HasMember("cid")) { + src = &camp["cid"]; + } + } + else { + if (imp.HasMember(fkey)) { + src = &imp[fkey]; + } + } + + if (src == nullptr) { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + int idx = FeatureIdxFromJsonValue(feature.c_str(), *src, &tables); + row[fi] = static_cast(idx); + } + + if (adsize_idx >= 0 && camp.HasMember("adsize") && camp["adsize"].IsArray()) { + const rapidjson::Value& adsize = camp["adsize"]; + for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { + const rapidjson::Value& adsize_item = adsize[ai]; + int mapped = FeatureIdxFromJsonValue("adsize", adsize_item, &tables); + row[static_cast(adsize_idx)] = static_cast(mapped); + err = AppendRowToNamedDoubleBuffers(&buffers, &counts, model_name, row, feature_sequence.size()); + if (err != nullptr) { + return err; + } + } + } + } + } + err = BuildMultiInferRequestDocument(buffers, counts, out_doc); + return err; + } +} +} // namespace triton::server +#endif // TRITON_ENABLE_MYSQL_ODBC diff --git a/src/transform.h b/src/transform.h new file mode 100644 index 0000000000..f481098d55 --- /dev/null +++ b/src/transform.h @@ -0,0 +1,70 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include + +#include "triton/core/tritonserver.h" + +namespace triton { namespace server { + +// Parses `json` into a document. If built with `TRITON_ENABLE_MYSQL_ODBC` and +// `server` is non-null and the root object contains `"imps"`, replaces +// `*out_doc` with a multi-infer request built from impressions; otherwise +// stores the parsed root in `*out_doc`. +// Returns nullptr on success, or a TRITONSERVER_Error that the caller must +// delete with TRITONSERVER_ErrorDelete on failure. +TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc); + +#ifdef TRITON_ENABLE_MYSQL_ODBC + +// Builds `*out_doc` as a `POST /v2/multi_infer` body from `doc` (imps/camps). +TRITONSERVER_Error* GenerateInputVectors( + const rapidjson::Document& doc, TRITONSERVER_Server* server, + rapidjson::Document* out_doc); + +TRITONSERVER_Error* GetReadyModelNames( + TRITONSERVER_Server* server, std::unordered_set* out); + +using NamedDoubleBuffers = std::unordered_map>; +using ModelNameToFeatureCount = std::unordered_map; + +TRITONSERVER_Error* AppendRowToNamedDoubleBuffers( + NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, + const std::string& vector_name, const std::vector& row, + size_t feature_count); + +TRITONSERVER_Error* BuildMultiInferRequestDocument( + const NamedDoubleBuffers& buffers, + const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc); + +#endif // TRITON_ENABLE_MYSQL_ODBC + +}} // namespace triton::server From 2de58d9e43fe4a7fde7daf1d760fa4feb1dc027c Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Tue, 16 Jun 2026 18:05:05 +0530 Subject: [PATCH 02/14] Removed unwanted files --- .github/workflows/pre-commit.yml | 37 +- .gitignore | 5 +- .pre-commit-config.yaml | 8 +- config/database_config.sample.json | 8 +- docs/client_guide/openai_readme.md | 344 ------------------ .../simple_identity/1/model.py | 26 -- 6 files changed, 14 insertions(+), 414 deletions(-) delete mode 100644 docs/client_guide/openai_readme.md diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 3e0db73aaa..6c33d435c9 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -1,8 +1,4 @@ -<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -======== -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ->>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,28 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py -import json - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """This model always returns the input that it has received.""" - - def initialize(self, args): - self.model_config = json.loads(args["model_config"]) - - def execute(self, requests): - """This function is called on inference request.""" - - responses = [] - for request in requests: - in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor_0 = pb_utils.Tensor("OUTPUT0", in_0.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor_0])) - return responses -======== name: pre-commit on: @@ -57,16 +31,15 @@ on: jobs: pre-commit: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5.0.0 + - uses: actions/checkout@v3 with: fetch-depth: 2 - name: Get modified files id: modified-files run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v6.0.0 - - uses: pre-commit/action@v3.0.1 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v3.0.0 with: extra_args: --files ${{ steps.modified-files.outputs.modified_files }} ->>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml diff --git a/.gitignore b/.gitignore index 01abc6eaab..e14c1671a5 100644 --- a/.gitignore +++ b/.gitignore @@ -10,12 +10,9 @@ test_results.txt artifacts cprofile *.prof -.venv -**/.venv # Test exclusions qa/L0_openai/openai tensorrtllm_models -tensorrtllm_mistral_models/ custom_tokenizer -replace-artifacts/ \ No newline at end of file +replace-artifacts/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd6320fe0d..663a36d631 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: -- repo: https://github.com/PyCQA/isort +- repo: https://github.com/timothycrosley/isort rev: 5.12.0 hooks: - id: isort @@ -36,7 +36,7 @@ repos: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 + rev: 5.0.4 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] @@ -57,7 +57,7 @@ repos: # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 + rev: v4.4.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs diff --git a/config/database_config.sample.json b/config/database_config.sample.json index db12e70189..061a159baf 100644 --- a/config/database_config.sample.json +++ b/config/database_config.sample.json @@ -2,12 +2,12 @@ "databaseIp" : "10.0.0.1", "databasePort" : 3306, "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver", - "primaryDSNName" : "PrimaryCentralisedMySQL", - "secondaryDSNName" : "SecondaryCentralisedMySQL", - "dsnUserName" : "kdbuser", + "primaryDSNName" : "primaryDSNName", + "secondaryDSNName" : "secondaryDSNName", + "dsnUserName" : "REPLACE_WITH_USER", "dsnUserPassword" : "REPLACE_WITH_SECRET", "queryRetryCount": 3, - "dcId": 1, + "dcId": "int values", "minPoolConnections": 2, "maxPoolConnections": 5 } diff --git a/docs/client_guide/openai_readme.md b/docs/client_guide/openai_readme.md deleted file mode 100644 index 53a9f461f4..0000000000 --- a/docs/client_guide/openai_readme.md +++ /dev/null @@ -1,344 +0,0 @@ - -# OpenAI-Compatible Frontend for Triton Inference Server (Beta) - -> [!NOTE] -> The OpenAI-Compatible API is currently in BETA. Its features and functionality -> are subject to change as we collect feedback. We're excited to hear any thoughts -> you have and what features you'd like to see! - -## Pre-requisites - -1. Docker + NVIDIA Container Runtime -2. A correctly configured `HF_TOKEN` for access to HuggingFace models. - - The current examples and testing primarily use the - [`meta-llama/Meta-Llama-3.1-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) - model, but you can manually bring your own models and adjust accordingly. - -## VLLM - -1. Launch the container and install dependencies: - - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. - - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to - access gated models, make sure this is set in your local environment if needed. - -```bash -docker run -it --net=host --gpus all --rm \ - -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ - -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 -``` - -2. Launch the OpenAI-compatible Triton Inference Server: -```bash -cd /opt/tritonserver/python/openai - -# NOTE: Adjust the --tokenizer based on the model being used -python3 openai_frontend/main.py --model-repository tests/vllm_models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct -``` - -
-Example output - -``` -... -+-----------------------+---------+--------+ -| Model | Version | Status | -+-----------------------+---------+--------+ -| llama-3.1-8b-instruct | 1 | READY | <- Correct Model Loaded in Triton -+-----------------------+---------+--------+ -... -Found model: name='llama-3.1-8b-instruct', backend='vllm' -[WARNING] Adding CORS for the following origins: ['http://localhost'] -INFO: Started server process [126] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully -``` - -
- -3. Send a `/v1/chat/completions` request: - - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. -```bash -MODEL="llama-3.1-8b-instruct" -curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ - "model": "'${MODEL}'", - "messages": [{"role": "user", "content": "Say this is a test!"}] -}' | jq -``` - -
-Example output - -```json -{ - "id": "cmpl-6930b296-7ef8-11ef-bdd1-107c6149ca79", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": - { - "content": "This is only a test.", - "tool_calls": null, - "role": "assistant", - "function_call": null - }, - "logprobs": null - } - ], - "created": 1727679085, - "model": "llama-3.1-8b-instruct", - "system_fingerprint": null, - "object": "chat.completion", - "usage": null -} -``` - -
- -4. Send a `/v1/completions` request: - - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. -```bash -MODEL="llama-3.1-8b-instruct" -curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ - "model": "'${MODEL}'", - "prompt": "Machine learning is" -}' | jq -``` - -
-Example output - -```json -{ - "id": "cmpl-d51df75c-7ef8-11ef-bdd1-107c6149ca79", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "logprobs": null, - "text": " a field of computer science that focuses on developing algorithms that allow computers to learn from" - } - ], - "created": 1727679266, - "model": "llama-3.1-8b-instruct", - "system_fingerprint": null, - "object": "text_completion", - "usage": null -} -``` - -
- -5. Benchmark with `genai-perf`: -- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) -- Or try using genai-perf from the [SDK container](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) - -```bash -MODEL="llama-3.1-8b-instruct" -TOKENIZER="meta-llama/Meta-Llama-3.1-8B-Instruct" -genai-perf profile \ - --model ${MODEL} \ - --tokenizer ${TOKENIZER} \ - --service-kind openai \ - --endpoint-type chat \ - --url localhost:9000 \ - --streaming -``` - -
-Example output - -``` -2024-10-14 22:43 [INFO] genai_perf.parser:82 - Profiling these models: llama-3.1-8b-instruct -2024-10-14 22:43 [INFO] genai_perf.wrapper:163 - Running Perf Analyzer : 'perf_analyzer -m llama-3.1-8b-instruct --async --input-data artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/inputs.json -i http --concurrency-range 1 --endpoint v1/chat/completions --service-kind openai -u localhost:9000 --measurement-interval 10000 --stability-percentage 999 --profile-export-file artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export.json' - NVIDIA GenAI-Perf | LLM Metrics -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ -┃ Statistic ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p75 ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ -│ Time to first token (ms) │ 71.66 │ 64.32 │ 86.52 │ 76.13 │ 74.92 │ 73.26 │ -│ Inter token latency (ms) │ 18.47 │ 18.25 │ 18.72 │ 18.67 │ 18.61 │ 18.53 │ -│ Request latency (ms) │ 348.00 │ 274.60 │ 362.27 │ 355.41 │ 352.29 │ 350.66 │ -│ Output sequence length │ 15.96 │ 12.00 │ 16.00 │ 16.00 │ 16.00 │ 16.00 │ -│ Input sequence length │ 549.66 │ 548.00 │ 551.00 │ 550.00 │ 550.00 │ 550.00 │ -│ Output token throughput (per sec) │ 45.84 │ N/A │ N/A │ N/A │ N/A │ N/A │ -│ Request throughput (per sec) │ 2.87 │ N/A │ N/A │ N/A │ N/A │ N/A │ -└───────────────────────────────────┴────────┴────────┴────────┴────────┴────────┴────────┘ -2024-10-14 22:44 [INFO] genai_perf.export_data.json_exporter:62 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.json -2024-10-14 22:44 [INFO] genai_perf.export_data.csv_exporter:71 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.csv -``` - -
- -6. Use the OpenAI python client directly: -```python -from openai import OpenAI - -client = OpenAI( - base_url="http://localhost:9000/v1", - api_key="EMPTY", -) - -model = "llama-3.1-8b-instruct" -completion = client.chat.completions.create( - model=model, - messages=[ - { - "role": "system", - "content": "You are a helpful assistant.", - }, - {"role": "user", "content": "What are LLMs?"}, - ], - max_tokens=256, -) - -print(completion.choices[0].message.content) -``` - -7. Run tests (NOTE: The server should not be running, the tests will handle starting/stopping the server as necessary): -```bash -cd /opt/tritonserver/python/openai/ -pip install -r requirements-test.txt - -pytest -v tests/ -``` - -## TensorRT-LLM - -0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: - - [Triton CLI](https://github.com/triton-inference-server/triton_cli/) - - [TRT-LLM Backend Quickstart](https://github.com/triton-inference-server/tensorrtllm_backend?tab=readme-ov-file#quick-start) - -1. Launch the container: - - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. - - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to - access gated models, make sure this is set in your local environment if needed. - -```bash -docker run -it --net=host --gpus all --rm \ - -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ - -e HF_TOKEN \ - -e TRTLLM_ORCHESTRATOR=1 \ - nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 -``` - -2. Install dependencies inside the container: -```bash -# Install python bindings for tritonserver and tritonfrontend -pip install /opt/tritonserver/python/triton*.whl - -# Install application requirements -git clone https://github.com/triton-inference-server/server.git -cd server/python/openai/ -pip install -r requirements.txt -``` - -2. Launch the OpenAI server: -```bash -# NOTE: Adjust the --tokenizer based on the model being used -python3 openai_frontend/main.py --model-repository path/to/models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct -``` - -3. Send a `/v1/chat/completions` request: - - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. -```bash -# MODEL should be the client-facing model name in your model repository for a pipeline like TRT-LLM. -# For example, this could also be "ensemble", or something like "gpt2" if generated from Triton CLI -MODEL="tensorrt_llm_bls" -curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ - "model": "'${MODEL}'", - "messages": [{"role": "user", "content": "Say this is a test!"}] -}' | jq -``` - -
-Example output - -```json -{ - "id": "cmpl-704c758c-8a84-11ef-b106-107c6149ca79", - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "It looks like you're testing the system!", - "tool_calls": null, - "role": "assistant", - "function_call": null - }, - "logprobs": null - } - ], - "created": 1728948689, - "model": "llama-3-8b-instruct", - "system_fingerprint": null, - "object": "chat.completion", - "usage": null -} -``` - -
- -The other examples should be the same as vLLM, except that you should set `MODEL="tensorrt_llm_bls"` or `MODEL="ensemble"`, -everywhere applicable as seen in the example request above. - -## KServe Frontends - -To support serving requests through both the OpenAI-Compatible and -KServe Predict v2 frontends to the same running Triton Inference Server, -the `tritonfrontend` python bindings are included for optional use in this -application as well. - -You can opt-in to including these additional frontends, assuming `tritonfrontend` -is installed, with `--enable-kserve-frontends` like below: - -``` -python3 openai_frontend/main.py \ - --model-repository tests/vllm_models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --enable-kserve-frontends -``` - -See `python3 openai_frontend/main.py --help` for more information on the -available arguments and default values. - -For more information on the `tritonfrontend` python bindings, see the docs -[here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md). - -## Model Parallelism Support - -- [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend)) - - ex: Configure `tensor_parallel_size: 2` in the - [model.json](https://github.com/triton-inference-server/vllm_backend/blob/main/samples/model_repository/vllm_model/1/model.json) -- [x] TensorRT-LLM ([Orchestrator Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#orchestrator-mode)) - - Set the following environment variable: `export TRTLLM_ORCHESTRATOR=1` -- [ ] TensorRT-LLM ([Leader Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#leader-mode)) - - Not currently supported diff --git a/docs/examples/model_repository/simple_identity/1/model.py b/docs/examples/model_repository/simple_identity/1/model.py index 3e0db73aaa..906c173892 100644 --- a/docs/examples/model_repository/simple_identity/1/model.py +++ b/docs/examples/model_repository/simple_identity/1/model.py @@ -1,8 +1,4 @@ -<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py # Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -======== -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ->>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,7 +24,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -<<<<<<<< HEAD:docs/examples/model_repository/simple_identity/1/model.py import json import triton_python_backend_utils as pb_utils @@ -49,24 +44,3 @@ def execute(self, requests): out_tensor_0 = pb_utils.Tensor("OUTPUT0", in_0.as_numpy()) responses.append(pb_utils.InferenceResponse([out_tensor_0])) return responses -======== -name: pre-commit - -on: - pull_request: - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5.0.0 - with: - fetch-depth: 2 - - name: Get modified files - id: modified-files - run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v6.0.0 - - uses: pre-commit/action@v3.0.1 - with: - extra_args: --files ${{ steps.modified-files.outputs.modified_files }} ->>>>>>>> r25.03_shantanu:.github/workflows/pre-commit.yml From 9a93bb01c149b488ef967be56d57ad1bc7cffb71 Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Wed, 17 Jun 2026 05:25:45 -0700 Subject: [PATCH 03/14] Inference score fix and response layout changes --- src/http_server.cc | 165 +++++++++++++++++++ src/http_server.h | 9 ++ src/multi_infer.cc | 387 +++++++++++++++++++++++++++++++++++++++------ src/transform.cc | 134 ++++++++++++---- src/transform.h | 36 ++--- 5 files changed, 638 insertions(+), 93 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index 845caf0569..c42fb61c91 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -4136,6 +4136,171 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( return nullptr; // success } +namespace { + +TRITONSERVER_Error* CopyTritonTensorPayloadToDoubles(const void* base, TRITONSERVER_DataType dtype, int64_t element_count, std::vector* out) +{ + out->resize(static_cast(element_count)); + switch (dtype) { + case TRITONSERVER_TYPE_BOOL: { + const uint8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = p[i] ? 1.0 : 0.0; + } + break; + } + case TRITONSERVER_TYPE_UINT8: { + const uint8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT16: { + const uint16_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT32: { + const uint32_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_UINT64: { + const uint64_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT8: { + const int8_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT16: { + const int16_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT32: { + const int32_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_INT64: { + const int64_t* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = static_cast(p[i]); + } + break; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + for (int64_t i = 0; i < element_count; ++i) { + (*out)[static_cast(i)] = p[i]; + } + break; + } + case TRITONSERVER_TYPE_FP16: + case TRITONSERVER_TYPE_BF16: + case TRITONSERVER_TYPE_BYTES: + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "tensor datatype not supported for direct multi_infer row extraction"); + case TRITONSERVER_TYPE_INVALID: + default: + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid or unsupported tensor datatype for row extraction"); + } + return nullptr; +} + +} // namespace + +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRowMajorDoubles(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out) +{ + rows_out->clear(); + if (expect_rows == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); + } + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); + + uint32_t output_count = 0; + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); + if (output_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "response has no outputs"); + } + + constexpr uint32_t kIdx = 0; + const char* cname = nullptr; + TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_INVALID; + const int64_t* shape = nullptr; + uint64_t dim_count = 0; + const void* base = nullptr; + size_t byte_size = 0; + TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; + int64_t memory_type_id = 0; + void* userp = nullptr; + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput(response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); + + auto* info = reinterpret_cast(userp); + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); + } + + int64_t element_count = 1; + for (uint64_t j = 0; j < dim_count; ++j) { + if (shape[j] < 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "negative dimension in output shape"); + } + element_count *= shape[j]; + } + if (element_count <= 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output has zero elements"); + } + if (element_count % static_cast(expect_rows) != 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output element count incompatible with expect_rows"); + } + + const int64_t elems_per_row = element_count / static_cast(expect_rows); + const size_t type_byte = TRITONSERVER_DataTypeByteSize(datatype); + const size_t expected_byte_size = static_cast(element_count) * type_byte; + if (expected_byte_size > byte_size) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "output byte_size too small for datatype/shape"); + } + + std::vector flat; + TRITONSERVER_Error* cerr = CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); + if (cerr != nullptr) { + return cerr; + } + + rows_out->resize(expect_rows); + for (size_t r = 0; r < expect_rows; ++r) { + const size_t off = r * static_cast(elems_per_row); + (*rows_out)[r].assign(flat.begin() + static_cast(off), flat.begin() + static_cast(off + static_cast(elems_per_row))); + } + return nullptr; +} + void HTTPAPIServer::InferRequestClass::SetResponseHeader( bool has_binary_data, size_t header_length) diff --git a/src/http_server.h b/src/http_server.h index 40e3a65263..f0e0a33a49 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -36,6 +36,7 @@ #include #include #include +#include #include "common.h" #include "data_compressor.h" @@ -327,6 +328,14 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out = nullptr); + // Reads output tensor 0 from `response` as row-major doubles without going + // through JSON. Same constraints as FinalizeResponse(json_only_out) on + // output 0: JSON-backed tensor, no classification. `expect_rows` must + // divide the total element count (leading batch rows). + TRITONSERVER_Error* ExtractFirstJsonOutputAsRowMajorDoubles( + TRITONSERVER_InferenceResponse* response, size_t expect_rows, + std::vector>* rows_out); + // Helper function to set infer response header in the form specified by // the endpoint protocol virtual void SetResponseHeader( diff --git a/src/multi_infer.cc b/src/multi_infer.cc index e7659446e5..7e135ce600 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -24,30 +24,54 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// POST /v2/multi_infer: full implementation (imps expansion, routing, and +// response folding) is compiled only when TRITON_ENABLE_MYSQL_ODBC is defined. +// Otherwise HandleMultiInfer responds with TRITONSERVER_ERROR_UNAVAILABLE. + #include "http_server.h" -#include "classification.h" #include "common.h" + +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "classification.h" #include "transform.h" +#include #include #include +#endif #include +#include +#include #include +#include #include #include #include +#include #include namespace triton { namespace server { #include "http_server_macros.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC + namespace { constexpr size_t kMaxMultiInferRequests = 16; +// Per batched infer row when folding multi_infer back to imps/camps (see +// imp_slot_routing in transform.cc). +struct ImpRouteCell { + int imp_idx{0}; + int camp_idx{0}; + int adsize_idx{0}; + int32_t cid{0}; + std::string mdl; +}; + int HttpCodeFromError(TRITONSERVER_Error* error) { if (error == nullptr) { return EVHTP_RES_OK; @@ -95,6 +119,12 @@ void AddContentTypeHeader(evhtp_request_t* req, const char* type) { evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); } +inline double RoundScore6(double x) +{ + constexpr double k = 1e6; + return std::floor(x * k) / k; +} + TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); { @@ -125,8 +155,7 @@ TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slo return nullptr; } -TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) -{ +TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) { ver_out->clear(); triton::common::TritonJson::Value mv; if (!slot.Find("model_version", &mv)) { @@ -148,6 +177,134 @@ TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Va return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); } +bool ParseImpSlotRoutingTable(const std::string& json, const size_t expect_slots, std::vector>* out) { + out->clear(); + if (json.empty() || expect_slots == 0) { + return false; + } + rapidjson::Document d; + d.Parse(json.data(), json.size()); + if (d.HasParseError() || !d.IsArray() || d.Size() != expect_slots) { + return false; + } + out->resize(d.Size()); + for (rapidjson::SizeType si = 0; si < d.Size(); ++si) { + const rapidjson::Value& slot = d[si]; + if (!slot.IsArray()) { + return false; + } + (*out)[si].reserve(slot.Size()); + for (rapidjson::SizeType ri = 0; ri < slot.Size(); ++ri) { + const rapidjson::Value& cell = slot[ri]; + if (!cell.IsObject() || !cell.HasMember("i") || !cell["i"].IsInt() || + !cell.HasMember("c") || !cell["c"].IsInt() || !cell.HasMember("a") || + !cell["a"].IsInt() || !cell.HasMember("cid") || !cell["cid"].IsInt() || + !cell.HasMember("mdl") || !cell["mdl"].IsString()) { + return false; + } + ImpRouteCell rc; + rc.imp_idx = cell["i"].GetInt(); + rc.camp_idx = cell["c"].GetInt(); + rc.adsize_idx = cell["a"].GetInt(); + rc.cid = static_cast(cell["cid"].GetInt()); + rc.mdl.assign(cell["mdl"].GetString(), cell["mdl"].GetStringLength()); + (*out)[si].push_back(std::move(rc)); + } + } + return true; +} + +bool TryBuildImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector>>& slot_rows, const int imp_count, rapidjson::Document* out) { + if (imp_count <= 0 || routing_slots.empty()) { + return false; + } + if (slot_rows.size() != routing_slots.size()) { + return false; + } + + struct CampAgg { + int32_t cid{0}; + std::string mdl; + std::map> by_adsize; + }; + std::map, CampAgg> agg; + + for (size_t si = 0; si < routing_slots.size(); ++si) { + const auto& slot_r = routing_slots[si]; + const size_t R = slot_r.size(); + if (si >= slot_rows.size()) { + return false; + } + const auto& row_vecs = slot_rows[si]; + if (row_vecs.size() != R) { + return false; + } + for (size_t ri = 0; ri < R; ++ri) { + const ImpRouteCell& rc = slot_r[ri]; + const auto key = std::make_pair(rc.imp_idx, rc.camp_idx); + CampAgg& ca = agg[key]; + if (ca.mdl.empty()) { + ca.cid = rc.cid; + ca.mdl = rc.mdl; + } else if (ca.cid != rc.cid || ca.mdl != rc.mdl) { + return false; + } + ca.by_adsize[rc.adsize_idx] = row_vecs[ri]; + } + } + + out->SetObject(); + auto& alloc = out->GetAllocator(); + rapidjson::Value imps_arr(rapidjson::kArrayType); + imps_arr.Reserve(static_cast(imp_count), alloc); + + for (int ii = 0; ii < imp_count; ++ii) { + std::vector camp_indices; + for (const auto& kv : agg) { + if (kv.first.first == ii) { + camp_indices.push_back(kv.first.second); + } + } + std::sort(camp_indices.begin(), camp_indices.end()); + camp_indices.erase(std::unique(camp_indices.begin(), camp_indices.end()), camp_indices.end()); + + rapidjson::Value camps_out(rapidjson::kArrayType); + for (int camp_j : camp_indices) { + const auto it = agg.find(std::make_pair(ii, camp_j)); + if (it == agg.end()) { + continue; + } + const CampAgg& ca = it->second; + rapidjson::Value camp_obj(rapidjson::kObjectType); + camp_obj.AddMember("cid", ca.cid, alloc); + camp_obj.AddMember("mdl", rapidjson::Value(ca.mdl.c_str(), static_cast(ca.mdl.size()), alloc).Move(),alloc); + rapidjson::Value score_arr(rapidjson::kArrayType); + for (const auto& ad_kv : ca.by_adsize) { + const std::vector& vec = ad_kv.second; + if (vec.size() == 1) { + score_arr.PushBack(RoundScore6(vec[0]), alloc); + } else { + rapidjson::Value inner(rapidjson::kArrayType); + inner.Reserve(static_cast(vec.size()), alloc); + for (double d : vec) { + inner.PushBack(RoundScore6(d), alloc); + } + score_arr.PushBack(inner, alloc); + } + } + camp_obj.AddMember("score", score_arr, alloc); + camps_out.PushBack(camp_obj, alloc); + } + + rapidjson::Value imp_wrap(rapidjson::kObjectType); + imp_wrap.AddMember("camps", camps_out, alloc); + imps_arr.PushBack(imp_wrap, alloc); + } + + out->AddMember("imps", imps_arr, alloc); + return true; +} + class MultiInferAggregator : public std::enable_shared_from_this { private: struct FinishPayload { @@ -155,16 +312,34 @@ class MultiInferAggregator : public std::enable_shared_from_this> irequests) + MultiInferAggregator(evhtp_request_t* req, size_t slot_count, evthr_t* reply_thread, + std::vector> irequests, + std::vector> imp_routing_slots = {}, + int imp_routing_imp_count = 0) : req_(req), n_(slot_count), reply_thread_(reply_thread), irequests_(std::move(irequests)), success_json_(slot_count), - error_text_(slot_count), have_error_(slot_count, 0){} + error_text_(slot_count), have_error_(slot_count, 0), + imp_routing_slots_(std::move(imp_routing_slots)), + imp_routing_imp_count_(imp_routing_imp_count) + { + slot_row_outputs_.assign(n_, {}); + } std::shared_ptr IrequestAt(size_t i) const { return irequests_[i]; } + bool WantsShardParsedRows() const + { + return imp_routing_imp_count_ > 0 && !imp_routing_slots_.empty(); + } + + size_t ExpectedRowsForSlot(size_t slot) const + { + return (slot < imp_routing_slots_.size()) ? imp_routing_slots_[slot].size() : 0; + } + void CancelAllSubRequests() { std::lock_guard lk(mu_); @@ -179,7 +354,7 @@ class MultiInferAggregator : public std::enable_shared_from_this> parsed_first_output = {}) { std::shared_ptr self; { std::lock_guard lk(mu_); @@ -197,6 +372,9 @@ class MultiInferAggregator : public std::enable_shared_from_this writer(sb); + shaped.Accept(writer); + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, sb.GetString(), sb.GetSize()); + evhtp_send_reply(req_, EVHTP_RES_OK); + evhtp_request_resume(req_); + return; + } + } + } + + bool any_shard_error = false; + for (size_t i = 0; i < n_; ++i) { + if (have_error_[i]) { + any_shard_error = true; + break; + } + } + if (any_shard_error) { + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value errors(root, triton::common::TritonJson::ValueType::ARRAY); + for (size_t i = 0; i < n_; ++i) { + TRITONSERVER_Error* ae = nullptr; + if (have_error_[i]) { + ae = errors.AppendString(error_text_[i]); + } else { + ae = errors.AppendString(""); + } + if (ae != nullptr) { + LOG_TRITONSERVER_ERROR(ae, "multi_infer: building errors array"); + TRITONSERVER_ErrorDelete(ae); + } + } + TRITONSERVER_Error* re = root.Add("errors", std::move(errors)); + if (re != nullptr) { + LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, re); + evhtp_send_reply(req_, HttpCodeFromError(re)); + TRITONSERVER_ErrorDelete(re); + evhtp_request_resume(req_); + return; + } + triton::common::TritonJson::WriteBuffer wb; + TRITONSERVER_Error* we = root.Write(&wb); + if (we != nullptr) { + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, we); + evhtp_send_reply(req_, HttpCodeFromError(we)); + TRITONSERVER_ErrorDelete(we); + } else { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + } + evhtp_request_resume(req_); + return; + } + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value responses(root, triton::common::TritonJson::ValueType::ARRAY); for (size_t i = 0; i < n_; ++i) { - if (have_error_[i]) { - triton::common::TritonJson::Value item(root, triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value item; + TRITONSERVER_Error* perr = item.Parse(success_json_[i].c_str(), success_json_[i].size()); + if (perr != nullptr) { + triton::common::TritonJson::Value wrap(root, triton::common::TritonJson::ValueType::OBJECT); triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); - TRITONSERVER_Error* ae = err_part.AddString("message", error_text_[i]); + TRITONSERVER_Error* ae = err_part.AddString("message", TRITONSERVER_ErrorMessage(perr)); + TRITONSERVER_ErrorDelete(perr); if (ae != nullptr) { - LOG_TRITONSERVER_ERROR(ae, "multi_infer: building error JSON"); TRITONSERVER_ErrorDelete(ae); } - TRITONSERVER_Error* be = item.Add("error", std::move(err_part)); + TRITONSERVER_Error* be = wrap.Add("error", std::move(err_part)); if (be != nullptr) { - LOG_TRITONSERVER_ERROR(be, "multi_infer: building error JSON"); TRITONSERVER_ErrorDelete(be); } + TRITONSERVER_Error* ce = responses.Append(std::move(wrap)); + if (ce != nullptr) { + TRITONSERVER_ErrorDelete(ce); + } + } else { TRITONSERVER_Error* ce = responses.Append(std::move(item)); if (ce != nullptr) { LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); TRITONSERVER_ErrorDelete(ce); } - } - else { - triton::common::TritonJson::Value item; - TRITONSERVER_Error* perr = item.Parse(success_json_[i].c_str(), success_json_[i].size()); - if (perr != nullptr) { - triton::common::TritonJson::Value wrap(root, triton::common::TritonJson::ValueType::OBJECT); - triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); - TRITONSERVER_Error* ae = err_part.AddString("message", TRITONSERVER_ErrorMessage(perr)); - TRITONSERVER_ErrorDelete(perr); - if (ae != nullptr) { - TRITONSERVER_ErrorDelete(ae); - } - TRITONSERVER_Error* be = wrap.Add("error", std::move(err_part)); - if (be != nullptr) { - TRITONSERVER_ErrorDelete(be); - } - TRITONSERVER_Error* ce = responses.Append(std::move(wrap)); - if (ce != nullptr) { - TRITONSERVER_ErrorDelete(ce); - } - } else { - TRITONSERVER_Error* ce = responses.Append(std::move(item)); - if (ce != nullptr) { - LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); - TRITONSERVER_ErrorDelete(ce); - } - } } } - TRITONSERVER_Error* re = root.Add("responses", std::move(responses)); + TRITONSERVER_Error* re = root.Add("errors", std::move(responses)); if (re != nullptr) { LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); AddContentTypeHeader(req_, "application/json"); @@ -307,6 +535,9 @@ class MultiInferAggregator : public std::enable_shared_from_this have_error_; bool cancel_sent_{false}; bool reply_scheduled_{false}; + std::vector> imp_routing_slots_; + std::vector>> slot_row_outputs_; + int imp_routing_imp_count_{0}; }; class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { @@ -327,10 +558,21 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { TRITONSERVER_Error* err = nullptr; evbuffer* shard_json = evbuffer_new(); + std::vector> pre_parsed_rows; if (infer_request->response_count_ != 1) { const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); } else if (response != nullptr) { + if (infer_request->aggregator_->WantsShardParsedRows()) { + const size_t nrows = infer_request->aggregator_->ExpectedRowsForSlot(infer_request->slot_); + if (nrows > 0u) { + TRITONSERVER_Error* ex_err = infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); + if (ex_err != nullptr) { + TRITONSERVER_ErrorDelete(ex_err); + pre_parsed_rows.clear(); + } + } + } err = infer_request->FinalizeResponse(response, shard_json); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { @@ -355,9 +597,8 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { if (err != nullptr) { infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); - } - else { - infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment); + } else { + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment, std::move(pre_parsed_rows)); } if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { @@ -400,6 +641,8 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; triton::common::TritonJson::Value root; + std::string imp_routing_meta; + int imp_routing_imp_count = 0; TRITONSERVER_Error* err = nullptr; const size_t body_len = evbuffer_get_length(body_buf); std::vector body_copy(body_len); @@ -414,6 +657,24 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { rapidjson::Document parsed; err = triton::server::ParseRequest(body_json, server_.get(), &parsed); if (err == nullptr) { + if (parsed.IsObject()) { + if (parsed.HasMember("imp_slot_routing") && parsed["imp_slot_routing"].IsArray()) { + rapidjson::StringBuffer rbuf; + rapidjson::Writer rw(rbuf); + parsed["imp_slot_routing"].Accept(rw); + imp_routing_meta.assign(rbuf.GetString(), rbuf.GetSize()); + parsed.RemoveMember("imp_slot_routing"); + } + if (parsed.HasMember("imp_routing_imp_count")) { + const rapidjson::Value& ic = parsed["imp_routing_imp_count"]; + if (ic.IsInt()) { + imp_routing_imp_count = ic.GetInt(); + } else if (ic.IsUint()) { + imp_routing_imp_count = static_cast(ic.GetUint()); + } + parsed.RemoveMember("imp_routing_imp_count"); + } + } rapidjson::StringBuffer sb; rapidjson::Writer writer(sb); parsed.Accept(writer); @@ -466,6 +727,16 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } + std::vector> imp_routing_table; + if (!imp_routing_meta.empty() && imp_routing_imp_count > 0) { + if (!ParseImpSlotRoutingTable(imp_routing_meta, n, &imp_routing_table)) { + imp_routing_table.clear(); + imp_routing_imp_count = 0; + } + } else { + imp_routing_imp_count = 0; + } + struct SlotPrep { std::string model_name; int64_t model_version{0}; @@ -574,7 +845,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { }); } - std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests); + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing_table), imp_routing_imp_count); std::vector> shard_holders; std::vector> release_holders; shard_holders.reserve(n); @@ -632,4 +903,32 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { } } +#else // !TRITON_ENABLE_MYSQL_ODBC + +void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) +{ + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + evhtp_request_pause(req); + static const char kMsg[] = "POST /v2/multi_infer requires a server built with TRITON_ENABLE_MYSQL_ODBC"; + auto* content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); + if (content_header != nullptr) { + evhtp_header_rm_and_free(req->headers_out, content_header); + } + evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, "application/json", 1, 1)); + triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); + response.AddStringRef("error", kMsg, sizeof(kMsg) - 1); + triton::common::TritonJson::WriteBuffer buffer_json; + TRITONSERVER_Error* we = response.Write(&buffer_json); + if (we != nullptr) { + TRITONSERVER_ErrorDelete(we); + evhtp_send_reply(req, EVHTP_RES_SERVERR); + } else { + evbuffer_add(req->buffer_out, buffer_json.Base(), buffer_json.Size()); + evhtp_send_reply(req, EVHTP_RES_SERVUNAVAIL); + } + evhtp_request_resume(req); +} + +#endif // TRITON_ENABLE_MYSQL_ODBC + }} // namespace triton::server diff --git a/src/transform.cc b/src/transform.cc index eb2f46e0f7..234a241cf0 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -31,12 +31,13 @@ #include #include #include +#include #include +#include #include #include #include #include -#include namespace { @@ -65,7 +66,7 @@ int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, } return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); } -} // namespace +} namespace triton { namespace server { @@ -82,8 +83,8 @@ TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* s return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); } - if (server != nullptr && doc.IsObject() && doc.HasMember("imps")) { - const rapidjson::Value& imps_member = doc["imps"]; + if (server != nullptr && doc.IsObject() && doc.HasMember(TRITON_BT_JSON_IMPS)) { + const rapidjson::Value& imps_member = doc[TRITON_BT_JSON_IMPS]; if (imps_member.IsArray()) { return GenerateInputVectors(doc, server, out_doc); } @@ -231,11 +232,54 @@ TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buf return nullptr; } +namespace { + +struct ImpRouteRow { + int imp_idx{0}; + int camp_idx{0}; + int adsize_idx{0}; + int32_t cid{0}; +}; + +void AddImpSlotRoutingMembers(const NamedDoubleBuffers& buffers, const std::unordered_map>& routes_by_model, int imp_count, rapidjson::Document* out_doc) { + std::vector names; + names.reserve(buffers.size()); + for (const auto& kv : buffers) { + names.push_back(kv.first); + } + std::sort(names.begin(), names.end()); + + auto& alloc = out_doc->GetAllocator(); + rapidjson::Value routing(rapidjson::kArrayType); + routing.Reserve(static_cast(names.size()), alloc); + for (const std::string& mn : names) { + rapidjson::Value slot(rapidjson::kArrayType); + auto it = routes_by_model.find(mn); + if (it != routes_by_model.end()) { + slot.Reserve(static_cast(it->second.size()), alloc); + for (const ImpRouteRow& r : it->second) { + rapidjson::Value o(rapidjson::kObjectType); + o.AddMember("i", r.imp_idx, alloc); + o.AddMember("c", r.camp_idx, alloc); + o.AddMember("a", r.adsize_idx, alloc); + o.AddMember(TRITON_BT_JSON_CID, r.cid, alloc); + o.AddMember("mdl", rapidjson::Value(mn.c_str(), static_cast(mn.size()), alloc).Move(), alloc); + slot.PushBack(o, alloc); + } + } + routing.PushBack(slot, alloc); + } + out_doc->AddMember("imp_slot_routing", routing, alloc); + out_doc->AddMember("imp_routing_imp_count", imp_count, alloc); +} + +} + TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { if (out_doc == nullptr || server == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); } - if (!doc.IsObject() || !doc.HasMember("imps") || !doc["imps"].IsArray()) { + if (!doc.IsObject() || !doc.HasMember(TRITON_BT_JSON_IMPS) || !doc[TRITON_BT_JSON_IMPS].IsArray()) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array"); } @@ -255,23 +299,24 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS NamedDoubleBuffers buffers; ModelNameToFeatureCount counts; + std::unordered_map> routes_by_model; - const rapidjson::Value& imps = doc["imps"]; + const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { const rapidjson::Value& imp = imps[ii]; if (!imp.IsObject()) { continue; } - if (!imp.HasMember("camps") || !imp["camps"].IsArray() || imp["camps"].Size() == 0) { + if (!imp.HasMember(TRITON_BT_JSON_CAMPS) || !imp[TRITON_BT_JSON_CAMPS].IsArray() || imp[TRITON_BT_JSON_CAMPS].Size() == 0) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each impression must include a non-empty camps array"); } - const rapidjson::Value& camps = imp["camps"]; + const rapidjson::Value& camps = imp[TRITON_BT_JSON_CAMPS]; for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { const rapidjson::Value& camp = camps[ci]; - if (!camp.IsObject() || !camp.HasMember("cid") || !camp["cid"].IsInt()) { + if (!camp.IsObject() || !camp.HasMember(TRITON_BT_JSON_CID) || !camp[TRITON_BT_JSON_CID].IsInt()) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each camp must be an object with integer 'cid'"); } - const int32_t campaign_id = camp["cid"].GetInt(); + const int32_t campaign_id = camp[TRITON_BT_JSON_CID].GetInt(); auto cmap_it = cmap->find(campaign_id); if(cmap_it == cmap->end()) { @@ -297,20 +342,20 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS const std::string& feature = feature_sequence[fi]; const char* fkey = feature.c_str(); - if (feature == "adsize") { + if (feature == TRITON_BT_FEATURE_ADSIZE) { adsize_idx = static_cast(fi); continue; } const rapidjson::Value* src = nullptr; - if (feature == "cookie" || feature == "rnk") { + if (feature == TRITON_BT_FEATURE_COOKIE || feature == TRITON_BT_FEATURE_RNK) { if (camp.HasMember(fkey)) { src = &camp[fkey]; } } - else if(feature == "campid") { - if (camp.HasMember("cid")) { - src = &camp["cid"]; + else if (feature == TRITON_BT_FEATURE_CAMPID) { + if (camp.HasMember(TRITON_BT_JSON_CID)) { + src = &camp[TRITON_BT_JSON_CID]; } } else { @@ -323,27 +368,56 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); } - int idx = FeatureIdxFromJsonValue(feature.c_str(), *src, &tables); - row[fi] = static_cast(idx); - } - if (adsize_idx >= 0 && camp.HasMember("adsize") && camp["adsize"].IsArray()) { - const rapidjson::Value& adsize = camp["adsize"]; - for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { - const rapidjson::Value& adsize_item = adsize[ai]; - int mapped = FeatureIdxFromJsonValue("adsize", adsize_item, &tables); - row[static_cast(adsize_idx)] = static_cast(mapped); - err = AppendRowToNamedDoubleBuffers(&buffers, &counts, model_name, row, feature_sequence.size()); - if (err != nullptr) { - return err; - } + const bool use_raw_numeric = (feature == TRITON_BT_FEATURE_UID) || + (feature == TRITON_BT_FEATURE_VIDEO_VPW) || (feature == TRITON_BT_FEATURE_VIDEO_VPH) || + (feature == TRITON_BT_FEATURE_MOBILEID) || (feature == TRITON_BT_FEATURE_VIEW) || + (feature == TRITON_BT_FEATURE_COOKIE) || (feature == TRITON_BT_FEATURE_RNK); + + if (use_raw_numeric) { + const rapidjson::Value& v = *src; + if (v.IsInt()) { + row[fi] = static_cast(v.GetInt()); + } else if (v.IsUint()) { + row[fi] = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + row[fi] = static_cast(v.GetInt64()); + } else if (v.IsUint64()) { + row[fi] = static_cast(v.GetUint64()); + } else if (v.IsDouble()) { + row[fi] = v.GetDouble(); + } else { + const std::string msg = std::string("feature '") + feature + "' must be a JSON number for raw passthrough"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg.c_str()); } + } else { + const int idx = FeatureIdxFromJsonValue(feature.c_str(), *src, &tables); + row[fi] = static_cast(idx); + } + } + if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && + camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { + const rapidjson::Value& adsize = camp[TRITON_BT_FEATURE_ADSIZE]; + for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { + const rapidjson::Value& adsize_item = adsize[ai]; + int mapped = FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables); + row[static_cast(adsize_idx)] = static_cast(mapped); + err = AppendRowToNamedDoubleBuffers(&buffers, &counts, model_name, row, feature_sequence.size()); + if (err != nullptr) { + return err; + } + routes_by_model[model_name].push_back(ImpRouteRow{ + static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); } } } - err = BuildMultiInferRequestDocument(buffers, counts, out_doc); + } + err = BuildMultiInferRequestDocument(buffers, counts, out_doc); + if (err != nullptr) { return err; } + AddImpSlotRoutingMembers(buffers, routes_by_model, static_cast(imps.Size()), out_doc); + return nullptr; } -} // namespace triton::server +} } // namespace triton::server #endif // TRITON_ENABLE_MYSQL_ODBC diff --git a/src/transform.h b/src/transform.h index f481098d55..c464dbdadf 100644 --- a/src/transform.h +++ b/src/transform.h @@ -35,35 +35,33 @@ namespace triton { namespace server { -// Parses `json` into a document. If built with `TRITON_ENABLE_MYSQL_ODBC` and -// `server` is non-null and the root object contains `"imps"`, replaces -// `*out_doc` with a multi-infer request built from impressions; otherwise -// stores the parsed root in `*out_doc`. -// Returns nullptr on success, or a TRITONSERVER_Error that the caller must -// delete with TRITONSERVER_ErrorDelete on failure. TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc); #ifdef TRITON_ENABLE_MYSQL_ODBC -// Builds `*out_doc` as a `POST /v2/multi_infer` body from `doc` (imps/camps). -TRITONSERVER_Error* GenerateInputVectors( - const rapidjson::Document& doc, TRITONSERVER_Server* server, - rapidjson::Document* out_doc); +#define TRITON_BT_FEATURE_ADSIZE "adsize" +#define TRITON_BT_FEATURE_COOKIE "cookie" +#define TRITON_BT_FEATURE_RNK "rnk" +#define TRITON_BT_FEATURE_CAMPID "campid" +#define TRITON_BT_JSON_IMPS "imps" +#define TRITON_BT_JSON_CAMPS "camps" +#define TRITON_BT_JSON_CID "cid" +#define TRITON_BT_FEATURE_UID "uid" +#define TRITON_BT_FEATURE_VIDEO_VPW "video_vpw" +#define TRITON_BT_FEATURE_VIDEO_VPH "video_vph" +#define TRITON_BT_FEATURE_MOBILEID "mobileid" +#define TRITON_BT_FEATURE_VIEW "view" -TRITONSERVER_Error* GetReadyModelNames( - TRITONSERVER_Server* server, std::unordered_set* out); +TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc); + +TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unordered_set* out); using NamedDoubleBuffers = std::unordered_map>; using ModelNameToFeatureCount = std::unordered_map; -TRITONSERVER_Error* AppendRowToNamedDoubleBuffers( - NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, - const std::string& vector_name, const std::vector& row, - size_t feature_count); +TRITONSERVER_Error* AppendRowToNamedDoubleBuffers(NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, const std::string& vector_name, const std::vector& row, size_t feature_count); -TRITONSERVER_Error* BuildMultiInferRequestDocument( - const NamedDoubleBuffers& buffers, - const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc); +TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buffers, const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc); #endif // TRITON_ENABLE_MYSQL_ODBC From d9fe870d005c4b1379d001c0d83f8aa038b93786 Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Wed, 17 Jun 2026 12:26:41 -0700 Subject: [PATCH 04/14] Added json parsing optimisations and string fixes --- src/http_server.cc | 9 +- src/multi_infer.cc | 405 +++++++++++++++++++++++---------------------- src/transform.cc | 4 +- src/transform.h | 2 +- 4 files changed, 216 insertions(+), 204 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index c42fb61c91..dca4d87ae5 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -3139,8 +3139,13 @@ HTTPAPIServer::DecompressBuffer( case DataCompressor::Type::DEFLATE: case DataCompressor::Type::GZIP: { *decompressed_buffer = evbuffer_new(); - RETURN_IF_ERR(DataCompressor::DecompressData( - compression_type, req->buffer_in, *decompressed_buffer)); + TRITONSERVER_Error* decompress_err = DataCompressor::DecompressData( + compression_type, req->buffer_in, *decompressed_buffer); + if (decompress_err != nullptr) { + evbuffer_free(*decompressed_buffer); + *decompressed_buffer = nullptr; + return decompress_err; + } break; } case DataCompressor::Type::UNKNOWN: { diff --git a/src/multi_infer.cc b/src/multi_infer.cc index 7e135ce600..a560901602 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -42,12 +42,13 @@ #endif #include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -125,71 +126,79 @@ inline double RoundScore6(double x) return std::floor(x * k) / k; } -TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { - *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); - { - triton::common::TritonJson::Value v; - if (slot.Find("id", &v)) { - RETURN_IF_ERR(infer_json->Add("id", std::move(v))); - } - } - { - triton::common::TritonJson::Value v; - if (!slot.Find("inputs", &v)) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); - } - RETURN_IF_ERR(infer_json->Add("inputs", std::move(v))); +TRITONSERVER_Error* GetModelVersionStringFromRapidSlot(const rapidjson::Value& slot, std::string* ver_out) { + ver_out->clear(); + const auto it = slot.FindMember("model_version"); + if (it == slot.MemberEnd()) { + return nullptr; } - { - triton::common::TritonJson::Value v; - if (slot.Find("outputs", &v)) { - RETURN_IF_ERR(infer_json->Add("outputs", std::move(v))); - } + const rapidjson::Value& mv = it->value; + if (mv.IsString()) { + ver_out->assign(mv.GetString(), mv.GetStringLength()); + return nullptr; } - { - triton::common::TritonJson::Value v; - if (slot.Find("parameters", &v)) { - RETURN_IF_ERR(infer_json->Add("parameters", std::move(v))); - } + if (mv.IsInt()) { + *ver_out = std::to_string(mv.GetInt()); + return nullptr; } - return nullptr; -} - -TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) { - ver_out->clear(); - triton::common::TritonJson::Value mv; - if (!slot.Find("model_version", &mv)) { + if (mv.IsUint()) { + *ver_out = std::to_string(mv.GetUint()); return nullptr; } - if (mv.IsString()) { - const char* s; - size_t len; - RETURN_IF_ERR(mv.AsString(&s, &len)); - ver_out->assign(s, len); + if (mv.IsInt64()) { + *ver_out = std::to_string(mv.GetInt64()); return nullptr; } - if (mv.IsNumber()) { - int64_t iv; - RETURN_IF_ERR(mv.AsInt(&iv)); - *ver_out = std::to_string(iv); + if (mv.IsUint64()) { + *ver_out = std::to_string(mv.GetUint64()); return nullptr; } return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); } -bool ParseImpSlotRoutingTable(const std::string& json, const size_t expect_slots, std::vector>* out) { +TRITONSERVER_Error* SerializeInferSlotBodyJsonFromRapid(const rapidjson::Value& slot, std::string* out_json) { + if (!slot.IsObject()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must be a JSON object"); + } + rapidjson::Document infer(rapidjson::kObjectType); + auto& alloc = infer.GetAllocator(); + if (const auto id_it = slot.FindMember("id"); id_it != slot.MemberEnd()) { + rapidjson::Value id_copy; + id_copy.CopyFrom(id_it->value, alloc); + infer.AddMember(rapidjson::StringRef("id"), id_copy, alloc); + } + const auto in_it = slot.FindMember("inputs"); + if (in_it == slot.MemberEnd() || !in_it->value.IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); + } + rapidjson::Value inputs_copy; + inputs_copy.CopyFrom(in_it->value, alloc); + infer.AddMember(rapidjson::StringRef("inputs"), inputs_copy, alloc); + if (const auto out_it = slot.FindMember("outputs"); out_it != slot.MemberEnd()) { + rapidjson::Value outputs_copy; + outputs_copy.CopyFrom(out_it->value, alloc); + infer.AddMember(rapidjson::StringRef("outputs"), outputs_copy, alloc); + } + if (const auto p_it = slot.FindMember("parameters"); p_it != slot.MemberEnd()) { + rapidjson::Value params_copy; + params_copy.CopyFrom(p_it->value, alloc); + infer.AddMember(rapidjson::StringRef("parameters"), params_copy, alloc); + } + rapidjson::StringBuffer sb; + rapidjson::Writer w(sb); + infer.Accept(w); + out_json->assign(sb.GetString(), sb.GetSize()); + return nullptr; +} + +bool ParseImpSlotRoutingTableFromValue(const rapidjson::Value& arr, const size_t expect_slots, std::vector>* out) { out->clear(); - if (json.empty() || expect_slots == 0) { - return false; - } - rapidjson::Document d; - d.Parse(json.data(), json.size()); - if (d.HasParseError() || !d.IsArray() || d.Size() != expect_slots) { + if (!arr.IsArray() || arr.Size() != expect_slots || expect_slots == 0) { return false; } - out->resize(d.Size()); - for (rapidjson::SizeType si = 0; si < d.Size(); ++si) { - const rapidjson::Value& slot = d[si]; + out->resize(arr.Size()); + for (rapidjson::SizeType si = 0; si < arr.Size(); ++si) { + const rapidjson::Value& slot = arr[si]; if (!slot.IsArray()) { return false; } @@ -214,6 +223,15 @@ bool ParseImpSlotRoutingTable(const std::string& json, const size_t expect_slots return true; } +inline uint64_t PackImpCampKey(int imp_idx, int camp_idx) { + return (static_cast(static_cast(imp_idx)) << 32) | + static_cast(static_cast(camp_idx)); +} + +inline int32_t ImpIdxFromPackedKey(uint64_t k) { + return static_cast(static_cast(k >> 32)); +} + bool TryBuildImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector>>& slot_rows, const int imp_count, rapidjson::Document* out) { if (imp_count <= 0 || routing_slots.empty()) { return false; @@ -227,7 +245,8 @@ bool TryBuildImpsShapedMultiInferResponse(const std::vector> by_adsize; }; - std::map, CampAgg> agg; + std::unordered_map agg; + agg.reserve(routing_slots.size() * 8); for (size_t si = 0; si < routing_slots.size(); ++si) { const auto& slot_r = routing_slots[si]; @@ -241,8 +260,8 @@ bool TryBuildImpsShapedMultiInferResponse(const std::vector> camps_ptr_per_imp(static_cast(imp_count)); + for (const auto& kv : agg) { + const int32_t imp = ImpIdxFromPackedKey(kv.first); + if (imp >= 0 && imp < imp_count) { + camps_ptr_per_imp[static_cast(imp)].push_back(&kv.second); + } + } + out->SetObject(); auto& alloc = out->GetAllocator(); rapidjson::Value imps_arr(rapidjson::kArrayType); imps_arr.Reserve(static_cast(imp_count), alloc); for (int ii = 0; ii < imp_count; ++ii) { - std::vector camp_indices; - for (const auto& kv : agg) { - if (kv.first.first == ii) { - camp_indices.push_back(kv.first.second); - } - } - std::sort(camp_indices.begin(), camp_indices.end()); - camp_indices.erase(std::unique(camp_indices.begin(), camp_indices.end()), camp_indices.end()); - rapidjson::Value camps_out(rapidjson::kArrayType); - for (int camp_j : camp_indices) { - const auto it = agg.find(std::make_pair(ii, camp_j)); - if (it == agg.end()) { - continue; - } - const CampAgg& ca = it->second; + for (const CampAgg* ca_ptr : camps_ptr_per_imp[static_cast(ii)]) { + const CampAgg& ca = *ca_ptr; rapidjson::Value camp_obj(rapidjson::kObjectType); camp_obj.AddMember("cid", ca.cid, alloc); - camp_obj.AddMember("mdl", rapidjson::Value(ca.mdl.c_str(), static_cast(ca.mdl.size()), alloc).Move(),alloc); + camp_obj.AddMember("mdl", rapidjson::Value(ca.mdl.c_str(), static_cast(ca.mdl.size()), alloc).Move(), alloc); rapidjson::Value score_arr(rapidjson::kArrayType); for (const auto& ad_kv : ca.by_adsize) { const std::vector& vec = ad_kv.second; @@ -340,13 +354,30 @@ class MultiInferAggregator : public std::enable_shared_from_this 0) { + return false; + } + } + return true; + } + void CancelAllSubRequests() { - std::lock_guard lk(mu_); - if (cancel_sent_) { + if (cancel_sent_.exchange(true, std::memory_order_acq_rel)) { return; } - cancel_sent_ = true; for (auto& ir : irequests_) { if (ir != nullptr) { LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); @@ -355,35 +386,30 @@ class MultiInferAggregator : public std::enable_shared_from_this> parsed_first_output = {}) { - std::shared_ptr self; - { - std::lock_guard lk(mu_); - if (finalize_err != nullptr) { - have_error_[slot] = 1; - error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); - TRITONSERVER_ErrorDelete(finalize_err); - if (!cancel_sent_) { - cancel_sent_ = true; - for (auto& ir : irequests_) { - if (ir != nullptr) { - LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); - } - } - } - } else { - success_json_[slot] = response_json; - if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { - slot_row_outputs_[slot] = std::move(parsed_first_output); - } - } - done_count_++; - if (done_count_ < n_ || reply_scheduled_) { - return; + if (finalize_err != nullptr) { + have_error_[slot] = 1; + error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); + TRITONSERVER_ErrorDelete(finalize_err); + } else { + success_json_[slot] = response_json; + if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { + slot_row_outputs_[slot] = std::move(parsed_first_output); } - reply_scheduled_ = true; - self = shared_from_this(); } + // Publish per-slot writes before the completion count; the last completer + // schedules WriteHttpReply which reads all slots. + std::atomic_thread_fence(std::memory_order_release); + + const size_t prev = done_count_.fetch_add(1, std::memory_order_acq_rel); + if (prev + 1 < n_) return; + + bool expected = false; + if (!reply_scheduled_.compare_exchange_strong(expected, true, std::memory_order_acq_rel, std::memory_order_relaxed)) { + return; + } + + std::shared_ptr self = shared_from_this(); auto* fp = new FinishPayload{std::move(self)}; evthr_defer(reply_thread_, FinishThunk, fp); } @@ -395,7 +421,9 @@ class MultiInferAggregator : public std::enable_shared_from_thisbuffer_out, kFoldErr, sizeof(kFoldErr) - 1); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + evhtp_request_resume(req_); + return; + } } } @@ -528,13 +566,12 @@ class MultiInferAggregator : public std::enable_shared_from_this> irequests_; - std::mutex mu_; - size_t done_count_{0}; + std::atomic done_count_{0}; std::vector success_json_; std::vector error_text_; std::vector have_error_; - bool cancel_sent_{false}; - bool reply_scheduled_{false}; + std::atomic cancel_sent_{false}; + std::atomic reply_scheduled_{false}; std::vector> imp_routing_slots_; std::vector>> slot_row_outputs_; int imp_routing_imp_count_{0}; @@ -557,23 +594,30 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { } TRITONSERVER_Error* err = nullptr; - evbuffer* shard_json = evbuffer_new(); + evbuffer* shard_json = nullptr; std::vector> pre_parsed_rows; if (infer_request->response_count_ != 1) { const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); } else if (response != nullptr) { + bool skip_shard_json = false; if (infer_request->aggregator_->WantsShardParsedRows()) { const size_t nrows = infer_request->aggregator_->ExpectedRowsForSlot(infer_request->slot_); if (nrows > 0u) { - TRITONSERVER_Error* ex_err = infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); - if (ex_err != nullptr) { + TRITONSERVER_Error* ex_err = + infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); + if (ex_err == nullptr) { + skip_shard_json = true; + } else { TRITONSERVER_ErrorDelete(ex_err); pre_parsed_rows.clear(); } } } - err = infer_request->FinalizeResponse(response, shard_json); + if (!skip_shard_json) { + shard_json = evbuffer_new(); + err = infer_request->FinalizeResponse(response, shard_json); + } #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp("INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); @@ -584,7 +628,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); std::string json_fragment; - if (err == nullptr) { + if (err == nullptr && shard_json != nullptr) { const size_t len = evbuffer_get_length(shard_json); if (len > 0) { const unsigned char* p = evbuffer_pullup(shard_json, -1); @@ -593,7 +637,9 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { } } } - evbuffer_free(shard_json); + if (shard_json != nullptr) { + evbuffer_free(shard_json); + } if (err != nullptr) { infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); @@ -640,47 +686,27 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; - triton::common::TritonJson::Value root; - std::string imp_routing_meta; + rapidjson::Document parsed; int imp_routing_imp_count = 0; TRITONSERVER_Error* err = nullptr; const size_t body_len = evbuffer_get_length(body_buf); - std::vector body_copy(body_len); + const char* body_ptr = ""; + std::vector body_copy; if (body_len > 0) { - const ssize_t nread = evbuffer_copyout(body_buf, body_copy.data(), body_len); - if (nread < 0 || static_cast(nread) != body_len) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); + if (const unsigned char* pulled = evbuffer_pullup(body_buf, -1); pulled != nullptr) { + body_ptr = reinterpret_cast(pulled); + } else { + body_copy.resize(body_len); + const ssize_t nread = evbuffer_copyout(body_buf, body_copy.data(), body_len); + if (nread < 0 || static_cast(nread) != body_len) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); + } else { + body_ptr = body_copy.data(); + } } } if (err == nullptr) { - const std::string body_json(body_copy.data(), body_len); - rapidjson::Document parsed; - err = triton::server::ParseRequest(body_json, server_.get(), &parsed); - if (err == nullptr) { - if (parsed.IsObject()) { - if (parsed.HasMember("imp_slot_routing") && parsed["imp_slot_routing"].IsArray()) { - rapidjson::StringBuffer rbuf; - rapidjson::Writer rw(rbuf); - parsed["imp_slot_routing"].Accept(rw); - imp_routing_meta.assign(rbuf.GetString(), rbuf.GetSize()); - parsed.RemoveMember("imp_slot_routing"); - } - if (parsed.HasMember("imp_routing_imp_count")) { - const rapidjson::Value& ic = parsed["imp_routing_imp_count"]; - if (ic.IsInt()) { - imp_routing_imp_count = ic.GetInt(); - } else if (ic.IsUint()) { - imp_routing_imp_count = static_cast(ic.GetUint()); - } - parsed.RemoveMember("imp_routing_imp_count"); - } - } - rapidjson::StringBuffer sb; - rapidjson::Writer writer(sb); - parsed.Accept(writer); - const std::string transformed(sb.GetString(), sb.GetSize()); - err = root.Parse(transformed.c_str(), transformed.size()); - } + err = triton::server::ParseRequest(body_ptr, body_len, server_.get(), &parsed); } if (decompressed_buffer != nullptr) { evbuffer_free(decompressed_buffer); @@ -695,8 +721,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - triton::common::TritonJson::Value requests; - if (!root.Find("requests", &requests)) { + if (!parsed.IsObject() || !parsed.HasMember("requests") || !parsed["requests"].IsArray()) { err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); @@ -706,7 +731,8 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - const size_t n = requests.ArraySize(); + const rapidjson::Value& requests = parsed["requests"]; + const size_t n = requests.Size(); if (n == 0) { err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); AddContentTypeHeader(req, "application/json"); @@ -727,9 +753,20 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } + if (parsed.IsObject() && parsed.HasMember("imp_routing_imp_count")) { + const rapidjson::Value& ic = parsed["imp_routing_imp_count"]; + if (ic.IsInt()) { + imp_routing_imp_count = ic.GetInt(); + } else if (ic.IsUint()) { + imp_routing_imp_count = static_cast(ic.GetUint()); + } + } + std::vector> imp_routing_table; - if (!imp_routing_meta.empty() && imp_routing_imp_count > 0) { - if (!ParseImpSlotRoutingTable(imp_routing_meta, n, &imp_routing_table)) { + const bool have_routing_array = + parsed.IsObject() && parsed.HasMember("imp_slot_routing") && parsed["imp_slot_routing"].IsArray(); + if (have_routing_array && imp_routing_imp_count > 0) { + if (!ParseImpSlotRoutingTableFromValue(parsed["imp_slot_routing"], n, &imp_routing_table)) { imp_routing_table.clear(); imp_routing_imp_count = 0; } @@ -737,6 +774,15 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { imp_routing_imp_count = 0; } + if (parsed.IsObject()) { + if (parsed.HasMember("imp_slot_routing")) { + parsed.RemoveMember("imp_slot_routing"); + } + if (parsed.HasMember("imp_routing_imp_count")) { + parsed.RemoveMember("imp_routing_imp_count"); + } + } + struct SlotPrep { std::string model_name; int64_t model_version{0}; @@ -745,10 +791,10 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { std::vector slots; slots.reserve(n); - for (size_t i = 0; i < n; ++i) { - triton::common::TritonJson::Value slot; - err = requests.At(i, &slot); - if (err != nullptr) { + for (rapidjson::SizeType i = 0; i < static_cast(n); ++i) { + const rapidjson::Value& slot = requests[i]; + if (!slot.IsObject()) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must be a JSON object"); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -756,10 +802,9 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } - const char* mn_c; - size_t mn_len; - err = slot.MemberAsString("model_name", &mn_c, &mn_len); - if (err != nullptr) { + const auto mn_it = slot.FindMember("model_name"); + if (mn_it == slot.MemberEnd() || !mn_it->value.IsString()) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must specify string 'model_name'"); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -768,9 +813,9 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } SlotPrep prep; - prep.model_name.assign(mn_c, mn_len); + prep.model_name.assign(mn_it->value.GetString(), mn_it->value.GetStringLength()); std::string ver_str; - err = GetModelVersionStringFromSlot(slot, &ver_str); + err = GetModelVersionStringFromRapidSlot(slot, &ver_str); if (err != nullptr) { AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); @@ -797,18 +842,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } - triton::common::TritonJson::Value infer_only; - err = CopyInferSlotBodyJson(slot, &infer_only); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - triton::common::TritonJson::WriteBuffer wb; - err = infer_only.Write(&wb); + err = SerializeInferSlotBodyJsonFromRapid(slot, &prep.infer_body_json); if (err != nullptr) { AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); @@ -817,7 +851,6 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } - prep.infer_body_json.assign(wb.Base(), wb.Size()); slots.push_back(std::move(prep)); } @@ -859,6 +892,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { err = GetInferenceHeaderLength(req, content_length, &header_length); if (err != nullptr) { aggregator->CancelAllSubRequests(); + evbuffer_free(body_i); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -902,33 +936,6 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { shard_holders[i].release(); } } - -#else // !TRITON_ENABLE_MYSQL_ODBC - -void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) -{ - RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); - evhtp_request_pause(req); - static const char kMsg[] = "POST /v2/multi_infer requires a server built with TRITON_ENABLE_MYSQL_ODBC"; - auto* content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); - if (content_header != nullptr) { - evhtp_header_rm_and_free(req->headers_out, content_header); - } - evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, "application/json", 1, 1)); - triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", kMsg, sizeof(kMsg) - 1); - triton::common::TritonJson::WriteBuffer buffer_json; - TRITONSERVER_Error* we = response.Write(&buffer_json); - if (we != nullptr) { - TRITONSERVER_ErrorDelete(we); - evhtp_send_reply(req, EVHTP_RES_SERVERR); - } else { - evbuffer_add(req->buffer_out, buffer_json.Base(), buffer_json.Size()); - evhtp_send_reply(req, EVHTP_RES_SERVUNAVAIL); - } - evhtp_request_resume(req); -} - #endif // TRITON_ENABLE_MYSQL_ODBC }} // namespace triton::server diff --git a/src/transform.cc b/src/transform.cc index 234a241cf0..468075e9e2 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -70,14 +70,14 @@ int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, namespace triton { namespace server { -TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { +TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { if (out_doc == nullptr) { return TRITONSERVER_ErrorNew( TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); } rapidjson::Document doc; - doc.Parse(json.data(), json.size()); + doc.Parse(json, json_len); if (doc.HasParseError()) { const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); diff --git a/src/transform.h b/src/transform.h index c464dbdadf..4afade1dcf 100644 --- a/src/transform.h +++ b/src/transform.h @@ -35,7 +35,7 @@ namespace triton { namespace server { -TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc); +TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc); #ifdef TRITON_ENABLE_MYSQL_ODBC From 3038b7ca775c71eb64e13f68815f5b0969542689 Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Tue, 23 Jun 2026 00:29:08 -0700 Subject: [PATCH 05/14] Made changes in Dockerfile --- Dockerfile | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 27454c0d38..9976e7ceda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,12 @@ # mkdir -p replace-artifacts # cp install/bin/tritonserver replace-artifacts/ # cp install/lib/libtritonserver.so replace-artifacts/ -# cp /etc/odbc.ini replace-artifacts/odbc.ini # optional if databaseIp is set (DSN unused) # Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for # optional JSON field "odbcDriverName" if the default fails. # +# ODBC DSN file (odbc.ini) and triton-dmconfig.json are not baked into the image; +# bind-mount host files to /etc/odbc.ini and /etc/triton-dmconfig.json at runtime (see Run). +# # Build (from repository root; match your Triton tag, e.g. r25.03): # docker build \ # --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \ @@ -32,13 +34,16 @@ # Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.: # --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04 # -# Run (mount a model repository and optionally override DM config): -# docker run --rm --gpus=all \ -# -p8000:8000 -p8001:8001 -p8002:8002 \ -# -v /path/to/model_repo:/models:ro \ -# -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \ +# Run (mount model repo; mount configs directly to /etc): +# docker run --name triton1 -d --net=host \ +# -v "/tmp/models:/models" \ +# -v "/etc/odbc.ini:/etc/odbc.ini:ro" \ +# -v "/etc/triton-dmconfig.json:/etc/triton-dmconfig.json:ro" \ # tritonserver:25.03-custom \ -# tritonserver --model-repository=/models +# tritonserver \ +# --model-repository=/models \ +# --model-control-mode explicit \ +# --http-port=4200 --grpc-port=4201 --metrics-port=4202 # # For CPU-only, drop --gpus=all and use a CPU/min base image. @@ -86,14 +91,6 @@ RUN set -eux; \ done; \ test -f /usr/lib/odbc/libmyodbc9w.so -# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json). -COPY replace-artifacts/odbc.ini /etc/odbc.ini -RUN chmod 644 /etc/odbc.ini - -# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro) -COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json -RUN chmod 644 /etc/triton-dmconfig.json - # Paths relative to the build context (repository root when building with ".") COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver COPY replace-artifacts/libtritonserver.so \ @@ -105,5 +102,4 @@ RUN chown 1000:1000 \ ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ && chmod 755 \ ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ - ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so - + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ No newline at end of file From b82ddbe285246fccf1e6761e52602ee0aaf77624 Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Wed, 24 Jun 2026 04:25:50 -0700 Subject: [PATCH 06/14] Fixed campaign ordering in request and response --- src/multi_infer.cc | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/multi_infer.cc b/src/multi_infer.cc index a560901602..13daee5020 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -122,8 +122,11 @@ void AddContentTypeHeader(evhtp_request_t* req, const char* type) { inline double RoundScore6(double x) { - constexpr double k = 1e6; - return std::floor(x * k) / k; + if (!std::isfinite(x)) { + return x; + } + const int64_t scaled = static_cast(std::floor(x * 1e6)); + return static_cast(scaled) / 1e6; } TRITONSERVER_Error* GetModelVersionStringFromRapidSlot(const rapidjson::Value& slot, std::string* ver_out) { @@ -232,6 +235,10 @@ inline int32_t ImpIdxFromPackedKey(uint64_t k) { return static_cast(static_cast(k >> 32)); } +inline int32_t CampIdxFromPackedKey(uint64_t k) { + return static_cast(static_cast(k & 0xffffffffu)); +} + bool TryBuildImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector>>& slot_rows, const int imp_count, rapidjson::Document* out) { if (imp_count <= 0 || routing_slots.empty()) { return false; @@ -272,13 +279,22 @@ bool TryBuildImpsShapedMultiInferResponse(const std::vector> camps_ptr_per_imp(static_cast(imp_count)); + struct CampAggEmitRef { + int32_t camp_idx{0}; + const CampAgg* agg{nullptr}; + }; + std::vector> camps_per_imp(static_cast(imp_count)); for (const auto& kv : agg) { const int32_t imp = ImpIdxFromPackedKey(kv.first); if (imp >= 0 && imp < imp_count) { - camps_ptr_per_imp[static_cast(imp)].push_back(&kv.second); + camps_per_imp[static_cast(imp)].push_back(CampAggEmitRef{CampIdxFromPackedKey(kv.first), &kv.second}); } } + for (auto& emits : camps_per_imp) { + std::sort(emits.begin(), emits.end(), [](const CampAggEmitRef& a, const CampAggEmitRef& b) { + return a.camp_idx < b.camp_idx; + }); + } out->SetObject(); auto& alloc = out->GetAllocator(); @@ -287,8 +303,8 @@ bool TryBuildImpsShapedMultiInferResponse(const std::vector(ii)]) { - const CampAgg& ca = *ca_ptr; + for (const CampAggEmitRef& emit : camps_per_imp[static_cast(ii)]) { + const CampAgg& ca = *emit.agg; rapidjson::Value camp_obj(rapidjson::kObjectType); camp_obj.AddMember("cid", ca.cid, alloc); camp_obj.AddMember("mdl", rapidjson::Value(ca.mdl.c_str(), static_cast(ca.mdl.size()), alloc).Move(), alloc); @@ -436,6 +452,7 @@ class MultiInferAggregator : public std::enable_shared_from_this writer(sb); + writer.SetMaxDecimalPlaces(6); shaped.Accept(writer); AddContentTypeHeader(req_, "application/json"); evbuffer_add(req_->buffer_out, sb.GetString(), sb.GetSize()); From 20e8f3b155cf8336baa3cf8f111a412d96f4c6ca Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Wed, 24 Jun 2026 13:21:19 -0700 Subject: [PATCH 07/14] Fixed missing rnk feature in feature mapping --- src/transform.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transform.cc b/src/transform.cc index 468075e9e2..ee92e45ad0 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -372,7 +372,7 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS const bool use_raw_numeric = (feature == TRITON_BT_FEATURE_UID) || (feature == TRITON_BT_FEATURE_VIDEO_VPW) || (feature == TRITON_BT_FEATURE_VIDEO_VPH) || (feature == TRITON_BT_FEATURE_MOBILEID) || (feature == TRITON_BT_FEATURE_VIEW) || - (feature == TRITON_BT_FEATURE_COOKIE) || (feature == TRITON_BT_FEATURE_RNK); + (feature == TRITON_BT_FEATURE_COOKIE); if (use_raw_numeric) { const rapidjson::Value& v = *src; From 835a493b38d5d3c67e4974bd58e195d326a0ea9f Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Thu, 25 Jun 2026 10:52:57 -0700 Subject: [PATCH 08/14] Made optimisations for json parsing and scheduling of requests --- src/CMakeLists.txt | 1 + src/http_error_json.h | 36 +++ src/http_server.cc | 164 +++++++++++-- src/http_server.h | 6 + src/http_server_macros.h | 3 +- src/main.cc | 2 + src/multi_infer.cc | 518 +++++++++++---------------------------- src/sagemaker_server.cc | 17 +- src/transform.cc | 432 ++++++++++++++++---------------- src/transform.h | 43 +++- 10 files changed, 587 insertions(+), 635 deletions(-) create mode 100644 src/http_error_json.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9445464ebc..5a892f3b31 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -370,6 +370,7 @@ if(${TRITON_ENABLE_HTTP} list(APPEND HTTP_ENDPOINT_HDRS http_server.h + http_error_json.h http_server_macros.h orca_http.h ) diff --git a/src/http_error_json.h b/src/http_error_json.h new file mode 100644 index 0000000000..4445c53560 --- /dev/null +++ b/src/http_error_json.h @@ -0,0 +1,36 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Lightweight {"error":"..."} serialization for HTTP error responses. +#pragma once + +#include +#include +#include + +#include + +#include "triton/core/tritonserver.h" + +namespace triton { namespace server { + +inline void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { + if (message == nullptr) { + message = ""; + } + + rapidjson::StringBuffer sb; + sb.Reserve(static_cast(std::strlen(message) + 16)); + rapidjson::Writer writer(sb); + writer.StartObject(); + writer.Key("error"); + writer.String(message); + writer.EndObject(); + + evbuffer_add(buffer, sb.GetString(), sb.GetSize()); +} + +inline void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { + EVBufferAddErrorJson(buffer, TRITONSERVER_ErrorMessage(err)); +} + +}} // namespace triton::server diff --git a/src/http_server.cc b/src/http_server.cc index dca4d87ae5..77c83f6731 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -42,6 +42,7 @@ #include #include "triton/common/triton_json.h" #include "classification.h" +#include "http_error_json.h" #include "http_server_macros.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* @@ -89,26 +90,6 @@ HttpCodeFromError(TRITONSERVER_Error* error) return EVHTP_RES_BADREQ; } -void -EVBufferAddErrorJson(evbuffer* buffer, const char* message) -{ - triton::common::TritonJson::Value response( - triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - -void -EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) -{ - const char* message = TRITONSERVER_ErrorMessage(err); - EVBufferAddErrorJson(buffer, message); -} - void AddContentTypeHeader(evhtp_request_t* req, const char* type) { @@ -4292,16 +4273,153 @@ TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRo return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "output byte_size too small for datatype/shape"); } + rows_out->resize(expect_rows); + const size_t epr = static_cast(elems_per_row); + + switch (datatype) { + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + if (epr == 1) { + for (size_t r = 0; r < expect_rows; ++r) { + (*rows_out)[r].resize(1); + (*rows_out)[r][0] = static_cast(p[r]); + } + } else { + for (size_t r = 0; r < expect_rows; ++r) { + auto& row = (*rows_out)[r]; + row.resize(epr); + const float* src = p + r * epr; + for (size_t j = 0; j < epr; ++j) { + row[j] = static_cast(src[j]); + } + } + } + return nullptr; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + if (epr == 1) { + for (size_t r = 0; r < expect_rows; ++r) { + (*rows_out)[r].resize(1); + (*rows_out)[r][0] = p[r]; + } + } else { + for (size_t r = 0; r < expect_rows; ++r) { + auto& row = (*rows_out)[r]; + row.resize(epr); + const double* src = p + r * epr; + for (size_t j = 0; j < epr; ++j) { + row[j] = src[j]; + } + } + } + return nullptr; + } + default: + break; + } + std::vector flat; TRITONSERVER_Error* cerr = CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); if (cerr != nullptr) { return cerr; } - rows_out->resize(expect_rows); for (size_t r = 0; r < expect_rows; ++r) { - const size_t off = r * static_cast(elems_per_row); - (*rows_out)[r].assign(flat.begin() + static_cast(off), flat.begin() + static_cast(off + static_cast(elems_per_row))); + const size_t off = r * epr; + (*rows_out)[r].assign(flat.begin() + static_cast(off), flat.begin() + static_cast(off + epr)); + } + return nullptr; +} + +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsScalars( + TRITONSERVER_InferenceResponse* response, size_t expect_rows, + std::vector* scores_out) +{ + scores_out->clear(); + if (expect_rows == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); + } + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); + + uint32_t output_count = 0; + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutputCount(response, &output_count)); + if (output_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "response has no outputs"); + } + + constexpr uint32_t kIdx = 0; + const char* cname = nullptr; + TRITONSERVER_DataType datatype = TRITONSERVER_TYPE_INVALID; + const int64_t* shape = nullptr; + uint64_t dim_count = 0; + const void* base = nullptr; + size_t byte_size = 0; + TRITONSERVER_MemoryType memory_type = TRITONSERVER_MEMORY_CPU; + int64_t memory_type_id = 0; + void* userp = nullptr; + + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( + response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, + &memory_type, &memory_type_id, &userp)); + + auto* info = reinterpret_cast(userp); + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); + } + + int64_t element_count = 1; + for (uint64_t j = 0; j < dim_count; ++j) { + if (shape[j] < 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "negative dimension in output shape"); + } + element_count *= shape[j]; + } + if (element_count <= 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output has zero elements"); + } + if (static_cast(element_count) != expect_rows) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "scalar extract requires one output element per batch row"); + } + + const size_t type_byte = TRITONSERVER_DataTypeByteSize(datatype); + const size_t expected_byte_size = static_cast(element_count) * type_byte; + if (expected_byte_size > byte_size) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "output byte_size too small for datatype/shape"); + } + + scores_out->resize(expect_rows); + switch (datatype) { + case TRITONSERVER_TYPE_FP32: { + const float* p = reinterpret_cast(base); + std::copy(p, p + expect_rows, scores_out->begin()); + return nullptr; + } + case TRITONSERVER_TYPE_FP64: { + const double* p = reinterpret_cast(base); + for (size_t r = 0; r < expect_rows; ++r) { + (*scores_out)[r] = static_cast(p[r]); + } + return nullptr; + } + default: + break; + } + + std::vector flat; + TRITONSERVER_Error* cerr = + CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); + if (cerr != nullptr) { + return cerr; + } + for (size_t r = 0; r < expect_rows; ++r) { + (*scores_out)[r] = static_cast(flat[r]); } return nullptr; } diff --git a/src/http_server.h b/src/http_server.h index f0e0a33a49..6dee9ed6d9 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -336,6 +336,12 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out); + // Like ExtractFirstJsonOutputAsRowMajorDoubles but returns one scalar per + // row when output 0 has a single element per batch row (common bt7 path). + TRITONSERVER_Error* ExtractFirstJsonOutputAsScalars( + TRITONSERVER_InferenceResponse* response, size_t expect_rows, + std::vector* scores_out); + // Helper function to set infer response header in the form specified by // the endpoint protocol virtual void SetResponseHeader( diff --git a/src/http_server_macros.h b/src/http_server_macros.h index 3cd47a1b14..df6f38c900 100644 --- a/src/http_server_macros.h +++ b/src/http_server_macros.h @@ -4,7 +4,8 @@ // and are not "in" a namespace; this header is included from within // triton::server for consistency with http_server.cc. // -// Prerequisites at expansion sites: HttpCodeFromError, EVBufferAddErrorJson, +// Prerequisites at expansion sites: HttpCodeFromError, EVBufferAddErrorJson +// (from http_error_json.h), // and (for RETURN_AND_RESPOND_IF_RESTRICTED) RespondIfRestricted must be // visible — typically from the same translation unit's anonymous namespace and // HTTPAPIServer member functions respectively. diff --git a/src/main.cc b/src/main.cc index 38b21ec648..c09f1a3bba 100644 --- a/src/main.cc +++ b/src/main.cc @@ -55,6 +55,7 @@ #include "database_config.h" #ifdef TRITON_ENABLE_MYSQL_ODBC #include "mysql_odbc_connection_pool.h" +#include "transform.h" #endif // TRITON_ENABLE_MYSQL_ODBC #include "triton_signal.h" @@ -621,6 +622,7 @@ main(int argc, char** argv) #ifdef TRITON_ENABLE_MYSQL_ODBC StartTritonModelsRefreshThread(); + FAIL_IF_ERR(triton::server::InitializeReadyModelNames(server_ptr), "initializing ready model names"); #endif // TRITON_ENABLE_MYSQL_ODBC // Wait until a signal terminates the server... diff --git a/src/multi_infer.cc b/src/multi_infer.cc index 13daee5020..dd805f43fa 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -24,9 +24,8 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// POST /v2/multi_infer: full implementation (imps expansion, routing, and -// response folding) is compiled only when TRITON_ENABLE_MYSQL_ODBC is defined. -// Otherwise HandleMultiInfer responds with TRITONSERVER_ERROR_UNAVAILABLE. +// POST /v2/multi_infer: imps-only path (feature transform, native tensors, +// response folding). Compiled only when TRITON_ENABLE_MYSQL_ODBC is defined. #include "http_server.h" @@ -46,13 +45,14 @@ #include #include #include -#include #include #include #include #include #include +#include "http_error_json.h" + namespace triton { namespace server { #include "http_server_macros.h" @@ -64,14 +64,7 @@ namespace { constexpr size_t kMaxMultiInferRequests = 16; // Per batched infer row when folding multi_infer back to imps/camps (see -// imp_slot_routing in transform.cc). -struct ImpRouteCell { - int imp_idx{0}; - int camp_idx{0}; - int adsize_idx{0}; - int32_t cid{0}; - std::string mdl; -}; +// ImpRouteRow in transform.h). int HttpCodeFromError(TRITONSERVER_Error* error) { if (error == nullptr) { @@ -96,32 +89,15 @@ int HttpCodeFromError(TRITONSERVER_Error* error) { return EVHTP_RES_BADREQ; } -void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { - triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - -void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { - const char* message = TRITONSERVER_ErrorMessage(err); - EVBufferAddErrorJson(buffer, message); -} - void AddContentTypeHeader(evhtp_request_t* req, const char* type) { auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); if (content_header) { evhtp_header_rm_and_free(req->headers_out, content_header); } - evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); } -inline double RoundScore6(double x) -{ +inline double RoundScore6(double x) { if (!std::isfinite(x)) { return x; } @@ -129,106 +105,38 @@ inline double RoundScore6(double x) return static_cast(scaled) / 1e6; } -TRITONSERVER_Error* GetModelVersionStringFromRapidSlot(const rapidjson::Value& slot, std::string* ver_out) { - ver_out->clear(); - const auto it = slot.FindMember("model_version"); - if (it == slot.MemberEnd()) { - return nullptr; +TRITONSERVER_Error* PopulateInferenceRequestFromNativeSlot(MultiInferNativeSlot slot, TRITONSERVER_InferenceRequest* irequest, HTTPAPIServer::InferRequestClass* infer_req) { + if (slot.feature_count == 0 || slot.input_tensor.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native slot has empty input tensor"); } - const rapidjson::Value& mv = it->value; - if (mv.IsString()) { - ver_out->assign(mv.GetString(), mv.GetStringLength()); - return nullptr; + if (slot.input_tensor.size() % (slot.feature_count * sizeof(float)) != 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native slot input size is not a multiple of feature_count"); } - if (mv.IsInt()) { - *ver_out = std::to_string(mv.GetInt()); - return nullptr; - } - if (mv.IsUint()) { - *ver_out = std::to_string(mv.GetUint()); - return nullptr; - } - if (mv.IsInt64()) { - *ver_out = std::to_string(mv.GetInt64()); - return nullptr; - } - if (mv.IsUint64()) { - *ver_out = std::to_string(mv.GetUint64()); - return nullptr; - } - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); -} -TRITONSERVER_Error* SerializeInferSlotBodyJsonFromRapid(const rapidjson::Value& slot, std::string* out_json) { - if (!slot.IsObject()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must be a JSON object"); - } - rapidjson::Document infer(rapidjson::kObjectType); - auto& alloc = infer.GetAllocator(); - if (const auto id_it = slot.FindMember("id"); id_it != slot.MemberEnd()) { - rapidjson::Value id_copy; - id_copy.CopyFrom(id_it->value, alloc); - infer.AddMember(rapidjson::StringRef("id"), id_copy, alloc); - } - const auto in_it = slot.FindMember("inputs"); - if (in_it == slot.MemberEnd() || !in_it->value.IsArray()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); - } - rapidjson::Value inputs_copy; - inputs_copy.CopyFrom(in_it->value, alloc); - infer.AddMember(rapidjson::StringRef("inputs"), inputs_copy, alloc); - if (const auto out_it = slot.FindMember("outputs"); out_it != slot.MemberEnd()) { - rapidjson::Value outputs_copy; - outputs_copy.CopyFrom(out_it->value, alloc); - infer.AddMember(rapidjson::StringRef("outputs"), outputs_copy, alloc); - } - if (const auto p_it = slot.FindMember("parameters"); p_it != slot.MemberEnd()) { - rapidjson::Value params_copy; - params_copy.CopyFrom(p_it->value, alloc); - infer.AddMember(rapidjson::StringRef("parameters"), params_copy, alloc); - } - rapidjson::StringBuffer sb; - rapidjson::Writer w(sb); - infer.Accept(w); - out_json->assign(sb.GetString(), sb.GetSize()); - return nullptr; -} + infer_req->alloc_payload_.default_output_kind_ = HTTPAPIServer::AllocPayload::OutputInfo::JSON; -bool ParseImpSlotRoutingTableFromValue(const rapidjson::Value& arr, const size_t expect_slots, std::vector>* out) { - out->clear(); - if (!arr.IsArray() || arr.Size() != expect_slots || expect_slots == 0) { - return false; - } - out->resize(arr.Size()); - for (rapidjson::SizeType si = 0; si < arr.Size(); ++si) { - const rapidjson::Value& slot = arr[si]; - if (!slot.IsArray()) { - return false; - } - (*out)[si].reserve(slot.Size()); - for (rapidjson::SizeType ri = 0; ri < slot.Size(); ++ri) { - const rapidjson::Value& cell = slot[ri]; - if (!cell.IsObject() || !cell.HasMember("i") || !cell["i"].IsInt() || - !cell.HasMember("c") || !cell["c"].IsInt() || !cell.HasMember("a") || - !cell["a"].IsInt() || !cell.HasMember("cid") || !cell["cid"].IsInt() || - !cell.HasMember("mdl") || !cell["mdl"].IsString()) { - return false; - } - ImpRouteCell rc; - rc.imp_idx = cell["i"].GetInt(); - rc.camp_idx = cell["c"].GetInt(); - rc.adsize_idx = cell["a"].GetInt(); - rc.cid = static_cast(cell["cid"].GetInt()); - rc.mdl.assign(cell["mdl"].GetString(), cell["mdl"].GetStringLength()); - (*out)[si].push_back(std::move(rc)); - } - } - return true; + const size_t row_bytes = slot.feature_count * sizeof(float); + const int64_t rows = static_cast(slot.input_tensor.size() / row_bytes); + const int64_t shape[2] = {rows, static_cast(slot.feature_count)}; + + constexpr const char* kInputName = "input__0"; + constexpr const char* kOutputName = "output__0"; + + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput(irequest, kInputName, TRITONSERVER_TYPE_FP32, shape, 2)); + + infer_req->serialized_data_.emplace_back(std::move(slot.input_tensor)); + std::vector& serialized = infer_req->serialized_data_.back(); + + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData(irequest, kInputName, serialized.data(), serialized.size(), TRITONSERVER_MEMORY_CPU, 0)); + + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, kOutputName)); + infer_req->alloc_payload_.output_map_.emplace(std::piecewise_construct, std::forward_as_tuple(kOutputName), std::forward_as_tuple(new HTTPAPIServer::AllocPayload::OutputInfo(HTTPAPIServer::AllocPayload::OutputInfo::JSON, 0))); + + return nullptr; } inline uint64_t PackImpCampKey(int imp_idx, int camp_idx) { - return (static_cast(static_cast(imp_idx)) << 32) | - static_cast(static_cast(camp_idx)); + return (static_cast(static_cast(imp_idx)) << 32) | static_cast(static_cast(camp_idx)); } inline int32_t ImpIdxFromPackedKey(uint64_t k) { @@ -239,50 +147,52 @@ inline int32_t CampIdxFromPackedKey(uint64_t k) { return static_cast(static_cast(k & 0xffffffffu)); } -bool TryBuildImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector>>& slot_rows, const int imp_count, rapidjson::Document* out) { - if (imp_count <= 0 || routing_slots.empty()) { +bool WriteImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector& slot_model_names, std::vector>>& slot_rows, const int imp_count, rapidjson::StringBuffer* sb) { + if (imp_count <= 0 || routing_slots.empty() || sb == nullptr) { return false; } - if (slot_rows.size() != routing_slots.size()) { + if (slot_rows.size() != routing_slots.size() || slot_model_names.size() != routing_slots.size()) { return false; } - struct CampAgg { - int32_t cid{0}; - std::string mdl; - std::map> by_adsize; - }; + struct CampAgg {int32_t cid{0}; const std::string* mdl{nullptr}; std::vector> by_adsize;}; std::unordered_map agg; - agg.reserve(routing_slots.size() * 8); + size_t total_rows = 0; + for (size_t si = 0; si < routing_slots.size(); ++si) { + total_rows += routing_slots[si].size(); + } + agg.reserve(total_rows); for (size_t si = 0; si < routing_slots.size(); ++si) { const auto& slot_r = routing_slots[si]; + const std::string& slot_mdl = slot_model_names[si]; const size_t R = slot_r.size(); if (si >= slot_rows.size()) { return false; } - const auto& row_vecs = slot_rows[si]; + auto& row_vecs = slot_rows[si]; if (row_vecs.size() != R) { return false; } for (size_t ri = 0; ri < R; ++ri) { - const ImpRouteCell& rc = slot_r[ri]; + const ImpRouteRow& rc = slot_r[ri]; const uint64_t pkey = PackImpCampKey(rc.imp_idx, rc.camp_idx); CampAgg& ca = agg[pkey]; - if (ca.mdl.empty()) { + if (ca.mdl == nullptr) { ca.cid = rc.cid; - ca.mdl = rc.mdl; - } else if (ca.cid != rc.cid || ca.mdl != rc.mdl) { + ca.mdl = &slot_mdl; + } else if (ca.cid != rc.cid || *ca.mdl != slot_mdl) { return false; } - ca.by_adsize[rc.adsize_idx] = row_vecs[ri]; + const size_t ad_idx = static_cast(rc.adsize_idx); + if (ad_idx >= ca.by_adsize.size()) { + ca.by_adsize.resize(ad_idx + 1); + } + ca.by_adsize[ad_idx] = std::move(row_vecs[ri]); } } - struct CampAggEmitRef { - int32_t camp_idx{0}; - const CampAgg* agg{nullptr}; - }; + struct CampAggEmitRef {int32_t camp_idx{0};const CampAgg* agg{nullptr};}; std::vector> camps_per_imp(static_cast(imp_count)); for (const auto& kv : agg) { const int32_t imp = ImpIdxFromPackedKey(kv.first); @@ -296,42 +206,49 @@ bool TryBuildImpsShapedMultiInferResponse(const std::vectorSetObject(); - auto& alloc = out->GetAllocator(); - rapidjson::Value imps_arr(rapidjson::kArrayType); - imps_arr.Reserve(static_cast(imp_count), alloc); + sb->Clear(); + sb->Reserve(static_cast(128 + total_rows * 96)); + rapidjson::Writer writer(*sb); + writer.SetMaxDecimalPlaces(6); + writer.StartObject(); + writer.Key("imps"); + writer.StartArray(); for (int ii = 0; ii < imp_count; ++ii) { - rapidjson::Value camps_out(rapidjson::kArrayType); + writer.StartObject(); + writer.Key("camps"); + writer.StartArray(); for (const CampAggEmitRef& emit : camps_per_imp[static_cast(ii)]) { const CampAgg& ca = *emit.agg; - rapidjson::Value camp_obj(rapidjson::kObjectType); - camp_obj.AddMember("cid", ca.cid, alloc); - camp_obj.AddMember("mdl", rapidjson::Value(ca.mdl.c_str(), static_cast(ca.mdl.size()), alloc).Move(), alloc); - rapidjson::Value score_arr(rapidjson::kArrayType); - for (const auto& ad_kv : ca.by_adsize) { - const std::vector& vec = ad_kv.second; + writer.StartObject(); + writer.Key("cid"); + writer.Int(ca.cid); + writer.Key("mdl"); + writer.String(ca.mdl->c_str(), static_cast(ca.mdl->size())); + writer.Key("score"); + writer.StartArray(); + for (const std::vector& vec : ca.by_adsize) { + if (vec.empty()) { + continue; + } if (vec.size() == 1) { - score_arr.PushBack(RoundScore6(vec[0]), alloc); + writer.Double(RoundScore6(vec[0])); } else { - rapidjson::Value inner(rapidjson::kArrayType); - inner.Reserve(static_cast(vec.size()), alloc); + writer.StartArray(); for (double d : vec) { - inner.PushBack(RoundScore6(d), alloc); + writer.Double(RoundScore6(d)); } - score_arr.PushBack(inner, alloc); + writer.EndArray(); } } - camp_obj.AddMember("score", score_arr, alloc); - camps_out.PushBack(camp_obj, alloc); + writer.EndArray(); + writer.EndObject(); } - - rapidjson::Value imp_wrap(rapidjson::kObjectType); - imp_wrap.AddMember("camps", camps_out, alloc); - imps_arr.PushBack(imp_wrap, alloc); + writer.EndArray(); + writer.EndObject(); } - - out->AddMember("imps", imps_arr, alloc); + writer.EndArray(); + writer.EndObject(); return true; } @@ -344,12 +261,14 @@ class MultiInferAggregator : public std::enable_shared_from_this> irequests, - std::vector> imp_routing_slots = {}, + std::vector> imp_routing_slots = {}, + std::vector slot_model_names = {}, int imp_routing_imp_count = 0) : req_(req), n_(slot_count), reply_thread_(reply_thread), irequests_(std::move(irequests)), success_json_(slot_count), error_text_(slot_count), have_error_(slot_count, 0), imp_routing_slots_(std::move(imp_routing_slots)), + slot_model_names_(std::move(slot_model_names)), imp_routing_imp_count_(imp_routing_imp_count) { slot_row_outputs_.assign(n_, {}); @@ -370,25 +289,6 @@ class MultiInferAggregator : public std::enable_shared_from_this 0) { - return false; - } - } - return true; - } - void CancelAllSubRequests() { if (cancel_sent_.exchange(true, std::memory_order_acq_rel)) { @@ -401,14 +301,19 @@ class MultiInferAggregator : public std::enable_shared_from_this> parsed_first_output = {}) { + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, const std::string& response_json, std::vector> parsed_first_output = {}, std::vector parsed_scalar_output = {}) { if (finalize_err != nullptr) { have_error_[slot] = 1; error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); TRITONSERVER_ErrorDelete(finalize_err); } else { success_json_[slot] = response_json; - if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { + if (!parsed_scalar_output.empty() && slot < slot_row_outputs_.size()) { + slot_row_outputs_[slot].resize(parsed_scalar_output.size()); + for (size_t r = 0; r < parsed_scalar_output.size(); ++r) { + slot_row_outputs_[slot][r] = {static_cast(parsed_scalar_output[r])}; + } + } else if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { slot_row_outputs_[slot] = std::move(parsed_first_output); } } @@ -448,28 +353,20 @@ class MultiInferAggregator : public std::enable_shared_from_this writer(sb); - writer.SetMaxDecimalPlaces(6); - shaped.Accept(writer); + rapidjson::StringBuffer sb; + if (WriteImpsShapedMultiInferResponse(imp_routing_slots_, slot_model_names_, slot_row_outputs_, imp_routing_imp_count_, &sb)) { AddContentTypeHeader(req_, "application/json"); evbuffer_add(req_->buffer_out, sb.GetString(), sb.GetSize()); evhtp_send_reply(req_, EVHTP_RES_OK); evhtp_request_resume(req_); return; } - // Folding failed after tensor-only shard path omitted per-shard JSON; do not - // fall through to legacy merge with empty fragments. - if (!LegacyMultiInferResponsesAvailable()) { - static const char kFoldErr[] = "{\"error\":{\"message\":\"failed to fold imps response; per-shard JSON was not retained for this request\"}}"; - AddContentTypeHeader(req_, "application/json"); - evbuffer_add(req_->buffer_out, kFoldErr, sizeof(kFoldErr) - 1); - evhtp_send_reply(req_, EVHTP_RES_BADREQ); - evhtp_request_resume(req_); - return; - } + static const char kFoldErr[] = "{\"error\":\"failed to fold imps response\"}"; + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, kFoldErr, sizeof(kFoldErr) - 1); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + evhtp_request_resume(req_); + return; } } @@ -521,60 +418,10 @@ class MultiInferAggregator : public std::enable_shared_from_thisbuffer_out, re); - evhtp_send_reply(req_, HttpCodeFromError(re)); - TRITONSERVER_ErrorDelete(re); - evhtp_request_resume(req_); - return; - } - - triton::common::TritonJson::WriteBuffer wb; - TRITONSERVER_Error* we = root.Write(&wb); - if (we != nullptr) { - AddContentTypeHeader(req_, "application/json"); - EVBufferAddErrorJson(req_->buffer_out, we); - evhtp_send_reply(req_, HttpCodeFromError(we)); - TRITONSERVER_ErrorDelete(we); - } - else { - AddContentTypeHeader(req_, "application/json"); - evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); - evhtp_send_reply(req_, EVHTP_RES_OK); - } + static const char kInternalErr[] = "{\"error\":\"unexpected multi_infer success path\"}"; + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, kInternalErr, sizeof(kInternalErr) - 1); + evhtp_send_reply(req_, EVHTP_RES_SERVERR); evhtp_request_resume(req_); } @@ -589,7 +436,8 @@ class MultiInferAggregator : public std::enable_shared_from_this have_error_; std::atomic cancel_sent_{false}; std::atomic reply_scheduled_{false}; - std::vector> imp_routing_slots_; + std::vector> imp_routing_slots_; + std::vector slot_model_names_; std::vector>> slot_row_outputs_; int imp_routing_imp_count_{0}; }; @@ -613,6 +461,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { TRITONSERVER_Error* err = nullptr; evbuffer* shard_json = nullptr; std::vector> pre_parsed_rows; + std::vector pre_parsed_scalars; if (infer_request->response_count_ != 1) { const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); @@ -621,13 +470,19 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { if (infer_request->aggregator_->WantsShardParsedRows()) { const size_t nrows = infer_request->aggregator_->ExpectedRowsForSlot(infer_request->slot_); if (nrows > 0u) { - TRITONSERVER_Error* ex_err = - infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); + TRITONSERVER_Error* ex_err = infer_request->ExtractFirstJsonOutputAsScalars(response, nrows, &pre_parsed_scalars); if (ex_err == nullptr) { skip_shard_json = true; } else { TRITONSERVER_ErrorDelete(ex_err); - pre_parsed_rows.clear(); + pre_parsed_scalars.clear(); + ex_err = infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); + if (ex_err == nullptr) { + skip_shard_json = true; + } else { + TRITONSERVER_ErrorDelete(ex_err); + pre_parsed_rows.clear(); + } } } } @@ -661,7 +516,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { if (err != nullptr) { infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); } else { - infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment, std::move(pre_parsed_rows)); + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment, std::move(pre_parsed_rows), std::move(pre_parsed_scalars)); } if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { @@ -704,7 +559,8 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; rapidjson::Document parsed; - int imp_routing_imp_count = 0; + ImpRoutingTable imp_routing; + std::vector native_slots; TRITONSERVER_Error* err = nullptr; const size_t body_len = evbuffer_get_length(body_buf); const char* body_ptr = ""; @@ -723,7 +579,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { } } if (err == nullptr) { - err = triton::server::ParseRequest(body_ptr, body_len, server_.get(), &parsed); + err = triton::server::ParseRequest(body_ptr, body_len, server_.get(), &parsed, &imp_routing, &native_slots); } if (decompressed_buffer != nullptr) { evbuffer_free(decompressed_buffer); @@ -738,20 +594,9 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - if (!parsed.IsObject() || !parsed.HasMember("requests") || !parsed["requests"].IsArray()) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - - const rapidjson::Value& requests = parsed["requests"]; - const size_t n = requests.Size(); + const size_t n = native_slots.size(); if (n == 0) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps transform produced no model inference slots"); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -770,86 +615,23 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - if (parsed.IsObject() && parsed.HasMember("imp_routing_imp_count")) { - const rapidjson::Value& ic = parsed["imp_routing_imp_count"]; - if (ic.IsInt()) { - imp_routing_imp_count = ic.GetInt(); - } else if (ic.IsUint()) { - imp_routing_imp_count = static_cast(ic.GetUint()); - } - } - - std::vector> imp_routing_table; - const bool have_routing_array = - parsed.IsObject() && parsed.HasMember("imp_slot_routing") && parsed["imp_slot_routing"].IsArray(); - if (have_routing_array && imp_routing_imp_count > 0) { - if (!ParseImpSlotRoutingTableFromValue(parsed["imp_slot_routing"], n, &imp_routing_table)) { - imp_routing_table.clear(); - imp_routing_imp_count = 0; - } - } else { - imp_routing_imp_count = 0; - } - - if (parsed.IsObject()) { - if (parsed.HasMember("imp_slot_routing")) { - parsed.RemoveMember("imp_slot_routing"); - } - if (parsed.HasMember("imp_routing_imp_count")) { - parsed.RemoveMember("imp_routing_imp_count"); - } + std::vector> imp_routing_table; + int imp_routing_imp_count = 0; + if (imp_routing.imp_count > 0 && imp_routing.slots.size() == n) { + imp_routing_table = std::move(imp_routing.slots); + imp_routing_imp_count = imp_routing.imp_count; } struct SlotPrep { std::string model_name; - int64_t model_version{0}; - std::string infer_body_json; + int64_t model_version{-1}; }; std::vector slots; slots.reserve(n); - for (rapidjson::SizeType i = 0; i < static_cast(n); ++i) { - const rapidjson::Value& slot = requests[i]; - if (!slot.IsObject()) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must be a JSON object"); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - const auto mn_it = slot.FindMember("model_name"); - if (mn_it == slot.MemberEnd() || !mn_it->value.IsString()) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must specify string 'model_name'"); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + for (size_t i = 0; i < n; ++i) { SlotPrep prep; - prep.model_name.assign(mn_it->value.GetString(), mn_it->value.GetStringLength()); - std::string ver_str; - err = GetModelVersionStringFromRapidSlot(slot, &ver_str); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - err = GetModelVersionFromString(ver_str, &prep.model_version); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + prep.model_name = native_slots[i].model_name; err = CheckTransactionPolicy(req, prep.model_name, prep.model_version); if (err != nullptr) { AddContentTypeHeader(req, "application/json"); @@ -859,15 +641,6 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } - err = SerializeInferSlotBodyJsonFromRapid(slot, &prep.infer_body_json); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } slots.push_back(std::move(prep)); } @@ -895,35 +668,24 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { }); } - std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing_table), imp_routing_imp_count); + std::vector slot_model_names; + slot_model_names.reserve(n); + for (const auto& slot : slots) { + slot_model_names.push_back(slot.model_name); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing_table),std::move(slot_model_names), imp_routing_imp_count); std::vector> shard_holders; std::vector> release_holders; shard_holders.reserve(n); release_holders.reserve(n); for (size_t i = 0; i < n; ++i) { - evbuffer* body_i = evbuffer_new(); - evbuffer_add(body_i, slots[i].infer_body_json.data(), slots[i].infer_body_json.size()); - const int32_t content_length = static_cast(evbuffer_get_length(body_i)); - size_t header_length = 0; - err = GetInferenceHeaderLength(req, content_length, &header_length); - if (err != nullptr) { - aggregator->CancelAllSubRequests(); - evbuffer_free(body_i); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); - err = EVRequestToTritonRequest(req, slots[i].model_name, irequests[i].get(), body_i, shard.get(), header_length); + err = PopulateInferenceRequestFromNativeSlot(std::move(native_slots[i]), irequests[i].get(), shard.get()); if (err != nullptr) { aggregator->CancelAllSubRequests(); - evbuffer_free(body_i); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -932,7 +694,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - auto rel = std::make_unique(irequests[i], body_i); + auto rel = std::make_unique(irequests[i], nullptr); err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); if (err != nullptr) { aggregator->CancelAllSubRequests(); diff --git a/src/sagemaker_server.cc b/src/sagemaker_server.cc index 52074f2b9d..75e9eb8363 100644 --- a/src/sagemaker_server.cc +++ b/src/sagemaker_server.cc @@ -25,6 +25,8 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "sagemaker_server.h" +#include "http_error_json.h" + namespace triton { namespace server { #define HTTP_RESPOND_IF_ERR(REQ, X) \ @@ -40,21 +42,6 @@ namespace triton { namespace server { namespace { -void -EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) -{ - const char* message = TRITONSERVER_ErrorMessage(err); - - triton::common::TritonJson::Value response( - triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - TRITONSERVER_Error* EVBufferToJson( triton::common::TritonJson::Value* document, evbuffer_iovec* v, int* v_idx, diff --git a/src/transform.cc b/src/transform.cc index ee92e45ad0..d87e8df9a8 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -13,7 +13,6 @@ // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, @@ -29,9 +28,11 @@ #include "mysql_odbc_connection_pool.h" #include #include +#include #include #include #include +#include #include #include #include @@ -41,6 +42,9 @@ namespace { +std::unordered_set g_ready_model_names; +std::atomic g_ready_models_valid{false}; + int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, const triton::server::FeatureMappingTables* tables) { if (tables == nullptr) { return -1; @@ -66,35 +70,107 @@ int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, } return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); } + +bool IsCampLevelFeature(const std::string& feature) { + return feature == TRITON_BT_FEATURE_COOKIE || feature == TRITON_BT_FEATURE_RNK || feature == TRITON_BT_FEATURE_CAMPID; } -namespace triton { namespace server { +bool UsesRawNumericFeature(const std::string& feature) { + return (feature == TRITON_BT_FEATURE_UID) || (feature == TRITON_BT_FEATURE_VIDEO_VPW) || (feature == TRITON_BT_FEATURE_VIDEO_VPH) || (feature == TRITON_BT_FEATURE_MOBILEID) || (feature == TRITON_BT_FEATURE_VIEW) || (feature == TRITON_BT_FEATURE_COOKIE); +} -TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { - if (out_doc == nullptr) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); +TRITONSERVER_Error* FillRawNumericFeature(const std::string& feature, const rapidjson::Value& v, float* out) { + if (v.IsInt()) { + *out = static_cast(v.GetInt()); + } else if (v.IsUint()) { + *out = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + *out = static_cast(v.GetInt64()); + } else if (v.IsUint64()) { + *out = static_cast(v.GetUint64()); + } else if (v.IsDouble()) { + *out = static_cast(v.GetDouble()); + } else { + const std::string msg = std::string("feature '") + feature + "' must be a JSON number for raw passthrough"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg.c_str()); } + return nullptr; +} - rapidjson::Document doc; - doc.Parse(json, json_len); - if (doc.HasParseError()) { - const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); - } +TRITONSERVER_Error* BuildImpBaseRow(const rapidjson::Value& imp, const std::vector& feature_sequence, const triton::server::FeatureMappingTables& tables, std::vector* out_row) { + out_row->assign(feature_sequence.size(), 0.0f); + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + if (feature == TRITON_BT_FEATURE_ADSIZE || IsCampLevelFeature(feature)) { + continue; + } - if (server != nullptr && doc.IsObject() && doc.HasMember(TRITON_BT_JSON_IMPS)) { - const rapidjson::Value& imps_member = doc[TRITON_BT_JSON_IMPS]; - if (imps_member.IsArray()) { - return GenerateInputVectors(doc, server, out_doc); + const char* fkey = feature.c_str(); + if (!imp.HasMember(fkey)) { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + const rapidjson::Value& v = imp[fkey]; + + TRITONSERVER_Error* err = nullptr; + if (UsesRawNumericFeature(feature)) { + err = FillRawNumericFeature(feature, v, &(*out_row)[fi]); + } else { + (*out_row)[fi] = static_cast(FeatureIdxFromJsonValue(feature.c_str(), v, &tables)); + } + if (err != nullptr) { + return err; } } + return nullptr; +} - *out_doc = std::move(doc); +TRITONSERVER_Error* FillCampFeaturesInRow(const rapidjson::Value& camp, int32_t campaign_id, const std::vector& feature_sequence, const triton::server::FeatureMappingTables& tables, std::vector* row) { + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + if (feature == TRITON_BT_FEATURE_ADSIZE) { + continue; + } + if (!IsCampLevelFeature(feature)) { + continue; + } + + const rapidjson::Value* src = nullptr; + if (feature == TRITON_BT_FEATURE_CAMPID) { + src = &camp[TRITON_BT_JSON_CID]; + } else if (camp.HasMember(feature.c_str())) { + src = &camp[feature.c_str()]; + } else { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + + TRITONSERVER_Error* err = nullptr; + if (UsesRawNumericFeature(feature)) { + err = FillRawNumericFeature(feature, *src, &(*row)[fi]); + } else { + (*row)[fi] = static_cast(FeatureIdxFromJsonValue(feature.c_str(), *src, &tables)); + } + if (err != nullptr) { + return err; + } + } return nullptr; } -TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unordered_set* out) { +void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) { + const size_t offset = tensor->size(); + tensor->resize(offset + row.size() * sizeof(float)); + std::memcpy(tensor->data() + offset, row.data(), row.size() * sizeof(float)); +} + +struct ModelSlotBuild { + size_t feature_count{0}; + std::vector tensor; + std::vector routes; +}; + +TRITONSERVER_Error* RefreshReadyModelNamesInto(std::unordered_set* out, TRITONSERVER_Server* server) { if (out == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output set pointer is null"); } @@ -141,142 +217,73 @@ TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unorder return nullptr; } +} + +namespace triton { namespace server { -TRITONSERVER_Error* AppendRowToNamedDoubleBuffers(NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, - const std::string& vector_name, const std::vector& row, size_t feature_count) { - if (buffers == nullptr || feature_counts == nullptr) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer pointers are null"); +TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server) { + if (server == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "server pointer is null"); } - if (row.size() != feature_count || feature_count == 0) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "row width does not match feature_count"); + if (g_ready_models_valid.load(std::memory_order_acquire)) { + return nullptr; } - - auto fc_it = feature_counts->find(vector_name); - if (fc_it == feature_counts->end()) { - (*feature_counts)[vector_name] = feature_count; - } else if (fc_it->second != feature_count) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); + TRITONSERVER_Error* err = RefreshReadyModelNamesInto(&g_ready_model_names, server); + if (err != nullptr) { + return err; } - - std::vector& buf = (*buffers)[vector_name]; - buf.insert(buf.end(), row.begin(), row.end()); + g_ready_models_valid.store(true, std::memory_order_release); return nullptr; } -TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buffers, const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc) { - if (out_doc == nullptr) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); +const std::unordered_set* ActiveReadyModelNames() { + if (!g_ready_models_valid.load(std::memory_order_acquire)) { + return nullptr; } + return &g_ready_model_names; +} - std::vector names; - names.reserve(buffers.size()); - for (const auto& kv : buffers) { - names.push_back(kv.first); +TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, std::vector* native_slots_out) { + if (out_doc == nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); } - std::sort(names.begin(), names.end()); - - rapidjson::Document doc(rapidjson::kObjectType); - auto& alloc = doc.GetAllocator(); - rapidjson::Value requests(rapidjson::kArrayType); - for (const std::string& model_name : names) { - auto bc_it = buffers.find(model_name); - auto fc_it = feature_counts.find(model_name); - if (bc_it == buffers.end() || fc_it == feature_counts.end()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "internal buffer map inconsistency"); - } - const size_t feature_count = fc_it->second; - if (feature_count == 0) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "zero feature_count for model"); - } - const std::vector& flat = bc_it->second; - if (flat.size() % feature_count != 0 || flat.empty()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); - } - const size_t rows = flat.size() / feature_count; + if (imp_routing_out != nullptr) { + imp_routing_out->imp_count = 0; + imp_routing_out->slots.clear(); + } + if (native_slots_out != nullptr) { + native_slots_out->clear(); + } - rapidjson::Value req(rapidjson::kObjectType); - req.AddMember("model_name", rapidjson::Value(model_name.c_str(), static_cast(model_name.size()), alloc).Move(), alloc); + rapidjson::Document doc; + doc.Parse(json, json_len); + if (doc.HasParseError()) { + const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); + } - rapidjson::Value data(rapidjson::kArrayType); - data.Reserve(static_cast(flat.size()), alloc); - for (double v : flat) { - data.PushBack(v, alloc); + if (server != nullptr && doc.IsObject() && doc.HasMember(TRITON_BT_JSON_IMPS)) { + const rapidjson::Value& imps_member = doc[TRITON_BT_JSON_IMPS]; + if (imps_member.IsArray()) { + if (native_slots_out == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native_slots_out is required for imps transform"); + } + return GenerateInputVectors(doc, server, out_doc, imp_routing_out, native_slots_out); } + } - rapidjson::Value shape(rapidjson::kArrayType); - shape.PushBack(static_cast(rows), alloc); - shape.PushBack(static_cast(feature_count), alloc); - - rapidjson::Value input0(rapidjson::kObjectType); - input0.AddMember("name", "input__0", alloc); - input0.AddMember("datatype", "FP32", alloc); - input0.AddMember("shape", shape, alloc); - input0.AddMember("data", data, alloc); - - rapidjson::Value inputs(rapidjson::kArrayType); - inputs.PushBack(input0, alloc); - - rapidjson::Value outputs(rapidjson::kArrayType); - rapidjson::Value out0(rapidjson::kObjectType); - out0.AddMember("name", "output__0", alloc); - outputs.PushBack(out0, alloc); - - req.AddMember("inputs", inputs, alloc); - req.AddMember("outputs", outputs, alloc); - requests.PushBack(req, alloc); + if (native_slots_out != nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "multi_infer expects a JSON object with an 'imps' array"); } - doc.AddMember("requests", requests, alloc); *out_doc = std::move(doc); return nullptr; } -namespace { - -struct ImpRouteRow { - int imp_idx{0}; - int camp_idx{0}; - int adsize_idx{0}; - int32_t cid{0}; -}; - -void AddImpSlotRoutingMembers(const NamedDoubleBuffers& buffers, const std::unordered_map>& routes_by_model, int imp_count, rapidjson::Document* out_doc) { - std::vector names; - names.reserve(buffers.size()); - for (const auto& kv : buffers) { - names.push_back(kv.first); - } - std::sort(names.begin(), names.end()); - - auto& alloc = out_doc->GetAllocator(); - rapidjson::Value routing(rapidjson::kArrayType); - routing.Reserve(static_cast(names.size()), alloc); - for (const std::string& mn : names) { - rapidjson::Value slot(rapidjson::kArrayType); - auto it = routes_by_model.find(mn); - if (it != routes_by_model.end()) { - slot.Reserve(static_cast(it->second.size()), alloc); - for (const ImpRouteRow& r : it->second) { - rapidjson::Value o(rapidjson::kObjectType); - o.AddMember("i", r.imp_idx, alloc); - o.AddMember("c", r.camp_idx, alloc); - o.AddMember("a", r.adsize_idx, alloc); - o.AddMember(TRITON_BT_JSON_CID, r.cid, alloc); - o.AddMember("mdl", rapidjson::Value(mn.c_str(), static_cast(mn.size()), alloc).Move(), alloc); - slot.PushBack(o, alloc); - } - } - routing.PushBack(slot, alloc); - } - out_doc->AddMember("imp_slot_routing", routing, alloc); - out_doc->AddMember("imp_routing_imp_count", imp_count, alloc); -} - -} - -TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { - if (out_doc == nullptr || server == nullptr) { +TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, std::vector* native_slots_out) { + if (out_doc == nullptr || server == nullptr || native_slots_out == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); } if (!doc.IsObject() || !doc.HasMember(TRITON_BT_JSON_IMPS) || !doc[TRITON_BT_JSON_IMPS].IsArray()) { @@ -288,20 +295,21 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); } - std::unordered_set ready_model_names; - TRITONSERVER_Error* err = GetReadyModelNames(server, &ready_model_names); - if (err != nullptr) { - return err; + const std::unordered_set* ready_model_names = ActiveReadyModelNames(); + if (ready_model_names == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "ready model names are not initialized"); } - if (ready_model_names.empty()) { + if (ready_model_names->empty()) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "no ready models reported by server"); } - NamedDoubleBuffers buffers; - ModelNameToFeatureCount counts; - std::unordered_map> routes_by_model; + std::unordered_map slots_by_model; const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; + std::vector row; + std::unordered_map> imp_base_by_model; + TRITONSERVER_Error* err = nullptr; + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { const rapidjson::Value& imp = imps[ii]; if (!imp.IsObject()) { @@ -311,6 +319,8 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each impression must include a non-empty camps array"); } const rapidjson::Value& camps = imp[TRITON_BT_JSON_CAMPS]; + imp_base_by_model.clear(); + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { const rapidjson::Value& camp = camps[ci]; if (!camp.IsObject() || !camp.HasMember(TRITON_BT_JSON_CID) || !camp[TRITON_BT_JSON_CID].IsInt()) { @@ -330,93 +340,95 @@ TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONS const std::vector& feature_sequence = cmap_it->second.feature_sequence; const FeatureMappingTables& tables = cmap_it->second.feature_mapping; - if (ready_model_names.find(model_name) == ready_model_names.end()) { + if (ready_model_names->find(model_name) == ready_model_names->end()) { const std::string not_ready = "model " + model_name + " not ready"; return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); } - std::vector row(feature_sequence.size(), 0.0); - int adsize_idx = -1; - - for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { - const std::string& feature = feature_sequence[fi]; - const char* fkey = feature.c_str(); - - if (feature == TRITON_BT_FEATURE_ADSIZE) { - adsize_idx = static_cast(fi); - continue; - } + ModelSlotBuild& slot = slots_by_model[model_name]; + const size_t feature_count = feature_sequence.size(); + if (slot.feature_count == 0) { + slot.feature_count = feature_count; + slot.tensor.reserve(camps.Size() * feature_count * sizeof(float)); + } else if (slot.feature_count != feature_count) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); + } - const rapidjson::Value* src = nullptr; - if (feature == TRITON_BT_FEATURE_COOKIE || feature == TRITON_BT_FEATURE_RNK) { - if (camp.HasMember(fkey)) { - src = &camp[fkey]; - } - } - else if (feature == TRITON_BT_FEATURE_CAMPID) { - if (camp.HasMember(TRITON_BT_JSON_CID)) { - src = &camp[TRITON_BT_JSON_CID]; - } - } - else { - if (imp.HasMember(fkey)) { - src = &imp[fkey]; - } + auto base_it = imp_base_by_model.find(model_name); + if (base_it == imp_base_by_model.end()) { + std::vector base_row; + err = BuildImpBaseRow(imp, feature_sequence, tables, &base_row); + if (err != nullptr) { + return err; } + base_it = imp_base_by_model.emplace(model_name, std::move(base_row)).first; + } - if (src == nullptr) { - const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); - } + row = base_it->second; + err = FillCampFeaturesInRow(camp, campaign_id, feature_sequence, tables, &row); + if (err != nullptr) { + return err; + } - const bool use_raw_numeric = (feature == TRITON_BT_FEATURE_UID) || - (feature == TRITON_BT_FEATURE_VIDEO_VPW) || (feature == TRITON_BT_FEATURE_VIDEO_VPH) || - (feature == TRITON_BT_FEATURE_MOBILEID) || (feature == TRITON_BT_FEATURE_VIEW) || - (feature == TRITON_BT_FEATURE_COOKIE); - - if (use_raw_numeric) { - const rapidjson::Value& v = *src; - if (v.IsInt()) { - row[fi] = static_cast(v.GetInt()); - } else if (v.IsUint()) { - row[fi] = static_cast(v.GetUint()); - } else if (v.IsInt64()) { - row[fi] = static_cast(v.GetInt64()); - } else if (v.IsUint64()) { - row[fi] = static_cast(v.GetUint64()); - } else if (v.IsDouble()) { - row[fi] = v.GetDouble(); - } else { - const std::string msg = std::string("feature '") + feature + "' must be a JSON number for raw passthrough"; - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg.c_str()); - } - } else { - const int idx = FeatureIdxFromJsonValue(feature.c_str(), *src, &tables); - row[fi] = static_cast(idx); + int adsize_idx = -1; + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + if (feature_sequence[fi] == TRITON_BT_FEATURE_ADSIZE) { + adsize_idx = static_cast(fi); + break; } } - if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && - camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { + + if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { const rapidjson::Value& adsize = camp[TRITON_BT_FEATURE_ADSIZE]; for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { const rapidjson::Value& adsize_item = adsize[ai]; - int mapped = FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables); - row[static_cast(adsize_idx)] = static_cast(mapped); - err = AppendRowToNamedDoubleBuffers(&buffers, &counts, model_name, row, feature_sequence.size()); - if (err != nullptr) { - return err; - } - routes_by_model[model_name].push_back(ImpRouteRow{ - static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); + row[static_cast(adsize_idx)] = static_cast(FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables)); + AppendFloatRowToTensor(&slot.tensor, row); + slot.routes.push_back(ImpRouteRow{static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); } } } } - err = BuildMultiInferRequestDocument(buffers, counts, out_doc); - if (err != nullptr) { - return err; + + std::vector model_names; + model_names.reserve(slots_by_model.size()); + for (const auto& kv : slots_by_model) { + model_names.push_back(kv.first); + } + std::sort(model_names.begin(), model_names.end()); + + native_slots_out->reserve(model_names.size()); + if (imp_routing_out != nullptr) { + imp_routing_out->imp_count = static_cast(imps.Size()); + imp_routing_out->slots.clear(); + imp_routing_out->slots.reserve(model_names.size()); } - AddImpSlotRoutingMembers(buffers, routes_by_model, static_cast(imps.Size()), out_doc); + + for (const std::string& model_name : model_names) { + auto it = slots_by_model.find(model_name); + if (it == slots_by_model.end()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "internal buffer map inconsistency"); + } + ModelSlotBuild& built = it->second; + if (built.feature_count == 0 || built.tensor.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "empty model buffer after transform"); + } + if (built.tensor.size() % (built.feature_count * sizeof(float)) != 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); + } + + MultiInferNativeSlot slot; + slot.model_name = model_name; + slot.feature_count = built.feature_count; + slot.input_tensor = std::move(built.tensor); + native_slots_out->push_back(std::move(slot)); + + if (imp_routing_out != nullptr) { + imp_routing_out->slots.push_back(std::move(built.routes)); + } + } + + out_doc->SetObject(); return nullptr; } } } // namespace triton::server diff --git a/src/transform.h b/src/transform.h index 4afade1dcf..f3a51e9952 100644 --- a/src/transform.h +++ b/src/transform.h @@ -35,8 +35,6 @@ namespace triton { namespace server { -TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc); - #ifdef TRITON_ENABLE_MYSQL_ODBC #define TRITON_BT_FEATURE_ADSIZE "adsize" @@ -52,16 +50,45 @@ TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER #define TRITON_BT_FEATURE_MOBILEID "mobileid" #define TRITON_BT_FEATURE_VIEW "view" -TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc); +// Per batched infer row when folding multi_infer results back to imps/camps. +struct ImpRouteRow { + int imp_idx{0}; + int camp_idx{0}; + int adsize_idx{0}; + int32_t cid{0}; +}; + +// In-memory routing for imps-shaped requests (not serialized into multi_infer JSON). +struct ImpRoutingTable { + int imp_count{0}; + // One vector per multi_infer request slot (sorted model name order). + std::vector> slots; +}; + +// Native FP32 tensor per multi_infer slot (imps transform path; no JSON tensor). +struct MultiInferNativeSlot { + std::string model_name; + size_t feature_count{0}; + // Row-major FP32 tensor bytes (feature_count * batch_rows * sizeof(float)). + std::vector input_tensor; +}; -TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unordered_set* out); +TRITONSERVER_Error* ParseRequest( + const char* json, size_t json_len, TRITONSERVER_Server* server, + rapidjson::Document* out_doc, + ImpRoutingTable* imp_routing_out = nullptr, + std::vector* native_slots_out = nullptr); -using NamedDoubleBuffers = std::unordered_map>; -using ModelNameToFeatureCount = std::unordered_map; +TRITONSERVER_Error* GenerateInputVectors( + const rapidjson::Document& doc, TRITONSERVER_Server* server, + rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, + std::vector* native_slots_out); -TRITONSERVER_Error* AppendRowToNamedDoubleBuffers(NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, const std::string& vector_name, const std::vector& row, size_t feature_count); +// Populate the ready-model snapshot once at process startup (after models are loaded). +TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server); -TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buffers, const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc); +// Lock-free read of the snapshot initialized by InitializeReadyModelNames. +const std::unordered_set* ActiveReadyModelNames(); #endif // TRITON_ENABLE_MYSQL_ODBC From 939ed0303ca6e8019603acfa0dac71f4a6a8cba3 Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Fri, 10 Jul 2026 16:20:50 +0530 Subject: [PATCH 09/14] Made optimisations and removing intermediate string operations --- src/http_server.cc | 139 ++++--- src/http_server.h | 29 +- src/multi_infer.cc | 609 ++++++++++++++++++++++-------- src/mysql_odbc_connection_pool.cc | 37 +- src/mysql_odbc_connection_pool.h | 7 + src/transform.cc | 215 ++++++----- src/transform.h | 33 +- 7 files changed, 734 insertions(+), 335 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index 77c83f6731..bf2d5296db 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -44,6 +44,10 @@ #include "classification.h" #include "http_error_json.h" #include "http_server_macros.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include +#endif #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ @@ -3206,6 +3210,36 @@ HTTPAPIServer::ScheduleInferAsync( return TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); } +TRITONSERVER_Error* HTTPAPIServer::FillMultiInferSlotTritonRequest(const std::string& model_name, triton::common::TritonJson::Value& infer_json, TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req) { + RETURN_IF_ERR(ParseJsonTritonRequestID(infer_json, irequest)); + RETURN_IF_ERR(ParseJsonTritonParams(infer_json, irequest, infer_req)); + int v_idx = 0; + RETURN_IF_ERR(ParseJsonTritonIO(infer_json, irequest, infer_req, model_name, nullptr, &v_idx, 0, 0)); + return nullptr; // success +} + +#ifdef TRITON_ENABLE_MYSQL_ODBC +TRITONSERVER_Error* HTTPAPIServer::AddJsonRequestedOutput(TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, const char* output_name, uint32_t class_cnt) { + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, output_name)); + infer_req->alloc_payload_.output_map_.emplace(std::piecewise_construct, std::forward_as_tuple(output_name), std::forward_as_tuple(new AllocPayload::OutputInfo(AllocPayload::OutputInfo::JSON, class_cnt))); + return nullptr; // success +} + +TRITONSERVER_Error* HTTPAPIServer::FillImpsTritonRequest(TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, ImpsInferSlot&& slot) { + const int64_t shape[] = {static_cast(slot.rows), static_cast(slot.feature_count)}; + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput(irequest, kImpsInputTensorName, TRITONSERVER_TYPE_FP32, shape, 2)); + + infer_req->serialized_data_.emplace_back(std::move(slot.input_tensor)); + std::vector& storage = infer_req->serialized_data_.back(); + const size_t byte_size = storage.size(); + + RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData(irequest, kImpsInputTensorName, (byte_size > 0) ? static_cast(storage.data()) : nullptr, byte_size, TRITONSERVER_MEMORY_CPU, 0 /* memory_type_id */)); + + RETURN_IF_ERR(AddJsonRequestedOutput(irequest, infer_req, kImpsOutputTensorName, 0)); + return nullptr; // success +} +#endif // TRITON_ENABLE_MYSQL_ODBC + void HTTPAPIServer::HandleGenerate( evhtp_request_t* req, const std::string& model_name, @@ -4063,11 +4097,27 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( RETURN_IF_ERR(response_json.Add("outputs", std::move(response_outputs))); + triton::common::TritonJson::WriteBuffer json_wb; + RETURN_IF_ERR(response_json.Write(&json_wb)); + const size_t json_byte_size = json_wb.Size(); + + if (json_only_out != nullptr) { + if (!ordered_buffers.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "multi_infer sub-request: binary outputs are not supported"); + } + evbuffer_add(json_only_out, json_wb.Base(), json_byte_size); + return nullptr; // success + } + + const bool use_identity_compression = (response_compression_type_ == DataCompressor::Type::IDENTITY) || (response_compression_type_ == DataCompressor::Type::UNKNOWN); + if (ordered_buffers.empty() && use_identity_compression) { + SetResponseHeader(false, json_byte_size); + evbuffer_add(req_->buffer_out, json_wb.Base(), json_byte_size); + return nullptr; // success + } + evbuffer* response_placeholder = evbuffer_new(); - // Write json metadata into response evbuffer - triton::common::TritonJson::WriteBuffer buffer; - RETURN_IF_ERR(response_json.Write(&buffer)); - evbuffer_add(response_placeholder, buffer.Base(), buffer.Size()); + evbuffer_add(response_placeholder, json_wb.Base(), json_byte_size); // If there is binary data write it next in the appropriate // order... also need the HTTP header when returning binary data. @@ -4078,45 +4128,28 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( } evbuffer* response_body = response_placeholder; - if (json_only_out == nullptr) { - switch (response_compression_type_) { - case DataCompressor::Type::DEFLATE: - case DataCompressor::Type::GZIP: { - auto compressed_buffer = evbuffer_new(); - auto err = DataCompressor::CompressData( - response_compression_type_, response_placeholder, - compressed_buffer); - if (err == nullptr) { - response_body = compressed_buffer; - evbuffer_free(response_placeholder); - } else { - // just log the compression error and return the uncompressed data - LOG_VERBOSE(1) << "unable to compress response: " - << TRITONSERVER_ErrorMessage(err); - TRITONSERVER_ErrorDelete(err); - evbuffer_free(compressed_buffer); - response_compression_type_ = DataCompressor::Type::IDENTITY; - } - break; + switch (response_compression_type_) { + case DataCompressor::Type::DEFLATE: + case DataCompressor::Type::GZIP: { + auto compressed_buffer = evbuffer_new(); + auto err = DataCompressor::CompressData(response_compression_type_, response_placeholder, compressed_buffer); + if (err == nullptr) { + response_body = compressed_buffer; + evbuffer_free(response_placeholder); + } else { + LOG_VERBOSE(1) << "unable to compress response: " << TRITONSERVER_ErrorMessage(err); + TRITONSERVER_ErrorDelete(err); + evbuffer_free(compressed_buffer); + response_compression_type_ = DataCompressor::Type::IDENTITY; } - case DataCompressor::Type::IDENTITY: - case DataCompressor::Type::UNKNOWN: - // Do nothing for other cases - break; - } - SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); - evbuffer_add_buffer(req_->buffer_out, response_body); - } else { - if (!ordered_buffers.empty()) { - evbuffer_free(response_placeholder); - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "multi_infer sub-request: binary outputs are not supported"); + break; } - evbuffer_add_buffer(json_only_out, response_body); + case DataCompressor::Type::IDENTITY: + case DataCompressor::Type::UNKNOWN: + break; } - // Destroy the evbuffer object as the data has been moved - // to HTTP response buffer (or json_only_out) + SetResponseHeader(!ordered_buffers.empty(), json_byte_size); + evbuffer_add_buffer(req_->buffer_out, response_body); evbuffer_free(response_body); return nullptr; // success @@ -4124,8 +4157,7 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( namespace { -TRITONSERVER_Error* CopyTritonTensorPayloadToDoubles(const void* base, TRITONSERVER_DataType dtype, int64_t element_count, std::vector* out) -{ +TRITONSERVER_Error* CopyTritonTensorPayloadToDoubles(const void* base, TRITONSERVER_DataType dtype, int64_t element_count, std::vector* out) { out->resize(static_cast(element_count)); switch (dtype) { case TRITONSERVER_TYPE_BOOL: { @@ -4218,8 +4250,7 @@ TRITONSERVER_Error* CopyTritonTensorPayloadToDoubles(const void* base, TRITONSER } // namespace -TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRowMajorDoubles(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out) -{ +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRowMajorDoubles(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out) { rows_out->clear(); if (expect_rows == 0) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); @@ -4332,10 +4363,7 @@ TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsRo return nullptr; } -TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsScalars( - TRITONSERVER_InferenceResponse* response, size_t expect_rows, - std::vector* scores_out) -{ +TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsScalars(TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector* scores_out) { scores_out->clear(); if (expect_rows == 0) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expect_rows must be positive"); @@ -4360,16 +4388,12 @@ TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsSc int64_t memory_type_id = 0; void* userp = nullptr; - RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput( - response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, - &memory_type, &memory_type_id, &userp)); + RETURN_IF_ERR(TRITONSERVER_InferenceResponseOutput(response, kIdx, &cname, &datatype, &shape, &dim_count, &base, &byte_size, &memory_type, &memory_type_id, &userp)); auto* info = reinterpret_cast(userp); if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || info->class_cnt_ > 0) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "output 0 must be plain JSON tensor (no shared memory / binary / classification)"); } int64_t element_count = 1; @@ -4383,9 +4407,7 @@ TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsSc return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output has zero elements"); } if (static_cast(element_count) != expect_rows) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_UNSUPPORTED, - "scalar extract requires one output element per batch row"); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNSUPPORTED, "scalar extract requires one output element per batch row"); } const size_t type_byte = TRITONSERVER_DataTypeByteSize(datatype); @@ -4413,8 +4435,7 @@ TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::ExtractFirstJsonOutputAsSc } std::vector flat; - TRITONSERVER_Error* cerr = - CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); + TRITONSERVER_Error* cerr = CopyTritonTensorPayloadToDoubles(base, datatype, element_count, &flat); if (cerr != nullptr) { return cerr; } diff --git a/src/http_server.h b/src/http_server.h index 6dee9ed6d9..61e0e5b30a 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -49,6 +49,10 @@ namespace triton { namespace server { +#ifdef TRITON_ENABLE_MYSQL_ODBC +struct ImpsInferSlot; +#endif + class MappingSchema { public: enum class Kind { @@ -328,16 +332,10 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out = nullptr); - // Reads output tensor 0 from `response` as row-major doubles without going - // through JSON. Same constraints as FinalizeResponse(json_only_out) on - // output 0: JSON-backed tensor, no classification. `expect_rows` must - // divide the total element count (leading batch rows). + // Direct tensor read for multi_infer imps folding (skips infer JSON build). TRITONSERVER_Error* ExtractFirstJsonOutputAsRowMajorDoubles( TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector>* rows_out); - - // Like ExtractFirstJsonOutputAsRowMajorDoubles but returns one scalar per - // row when output 0 has a single element per batch row (common bt7 path). TRITONSERVER_Error* ExtractFirstJsonOutputAsScalars( TRITONSERVER_InferenceResponse* response, size_t expect_rows, std::vector* scores_out); @@ -676,6 +674,23 @@ class HTTPAPIServer : public HTTPServer { triton::common::TritonJson::Value& request_json, TRITONSERVER_InferenceRequest* irequest); + // Fills irequest from a multi_infer slot object (inputs/outputs/id/parameters). + // JSON tensor data only; no trailing binary block (v/n/header_length unused). + TRITONSERVER_Error* FillMultiInferSlotTritonRequest( + const std::string& model_name, + triton::common::TritonJson::Value& infer_json, + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req); + +#ifdef TRITON_ENABLE_MYSQL_ODBC + static TRITONSERVER_Error* AddJsonRequestedOutput( + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, + const char* output_name, uint32_t class_cnt = 0); + + static TRITONSERVER_Error* FillImpsTritonRequest( + TRITONSERVER_InferenceRequest* irequest, InferRequestClass* infer_req, + ImpsInferSlot&& slot); +#endif // TRITON_ENABLE_MYSQL_ODBC + std::shared_ptr server_; // Storing server metadata as it is consistent during server running diff --git a/src/multi_infer.cc b/src/multi_infer.cc index dd805f43fa..302e4e783c 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -24,17 +24,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// POST /v2/multi_infer: imps-only path (feature transform, native tensors, -// response folding). Compiled only when TRITON_ENABLE_MYSQL_ODBC is defined. - #include "http_server.h" +#include "classification.h" #include "common.h" - #ifdef TRITON_ENABLE_MYSQL_ODBC -#include "classification.h" #include "transform.h" - #include #include #include @@ -43,29 +38,22 @@ #include #include #include -#include +#include #include #include -#include +#include #include -#include +#include #include -#include "http_error_json.h" - namespace triton { namespace server { #include "http_server_macros.h" -#ifdef TRITON_ENABLE_MYSQL_ODBC - namespace { constexpr size_t kMaxMultiInferRequests = 16; -// Per batched infer row when folding multi_infer back to imps/camps (see -// ImpRouteRow in transform.h). - int HttpCodeFromError(TRITONSERVER_Error* error) { if (error == nullptr) { return EVHTP_RES_OK; @@ -89,50 +77,130 @@ int HttpCodeFromError(TRITONSERVER_Error* error) { return EVHTP_RES_BADREQ; } +void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { + triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); + response.AddStringRef("error", message, strlen(message)); + + triton::common::TritonJson::WriteBuffer buffer_json; + response.Write(&buffer_json); + + evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); +} + +void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { + const char* message = TRITONSERVER_ErrorMessage(err); + EVBufferAddErrorJson(buffer, message); +} + void AddContentTypeHeader(evhtp_request_t* req, const char* type) { auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); if (content_header) { evhtp_header_rm_and_free(req->headers_out, content_header); } + evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); } -inline double RoundScore6(double x) { - if (!std::isfinite(x)) { - return x; +void AppendJsonEscaped(std::string* out, const std::string& value) +{ + out->reserve(out->size() + value.size() + 8); + for (char c : value) { + switch (c) { + case '"': + out->append("\\\""); + break; + case '\\': + out->append("\\\\"); + break; + case '\b': + out->append("\\b"); + break; + case '\f': + out->append("\\f"); + break; + case '\n': + out->append("\\n"); + break; + case '\r': + out->append("\\r"); + break; + case '\t': + out->append("\\t"); + break; + default: + if (static_cast(c) < 0x20) { + char buf[7]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(c)); + out->append(buf); + } else { + out->push_back(c); + } + break; + } } - const int64_t scaled = static_cast(std::floor(x * 1e6)); - return static_cast(scaled) / 1e6; } -TRITONSERVER_Error* PopulateInferenceRequestFromNativeSlot(MultiInferNativeSlot slot, TRITONSERVER_InferenceRequest* irequest, HTTPAPIServer::InferRequestClass* infer_req) { - if (slot.feature_count == 0 || slot.input_tensor.empty()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native slot has empty input tensor"); +TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { + *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); + { + triton::common::TritonJson::Value v; + if (slot.Find("id", &v)) { + RETURN_IF_ERR(infer_json->Add("id", std::move(v))); + } } - if (slot.input_tensor.size() % (slot.feature_count * sizeof(float)) != 0) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native slot input size is not a multiple of feature_count"); + { + triton::common::TritonJson::Value v; + if (!slot.Find("inputs", &v)) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); + } + RETURN_IF_ERR(infer_json->Add("inputs", std::move(v))); } + { + triton::common::TritonJson::Value v; + if (slot.Find("outputs", &v)) { + RETURN_IF_ERR(infer_json->Add("outputs", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (slot.Find("parameters", &v)) { + RETURN_IF_ERR(infer_json->Add("parameters", std::move(v))); + } + } + return nullptr; +} - infer_req->alloc_payload_.default_output_kind_ = HTTPAPIServer::AllocPayload::OutputInfo::JSON; - - const size_t row_bytes = slot.feature_count * sizeof(float); - const int64_t rows = static_cast(slot.input_tensor.size() / row_bytes); - const int64_t shape[2] = {rows, static_cast(slot.feature_count)}; - - constexpr const char* kInputName = "input__0"; - constexpr const char* kOutputName = "output__0"; - - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddInput(irequest, kInputName, TRITONSERVER_TYPE_FP32, shape, 2)); - - infer_req->serialized_data_.emplace_back(std::move(slot.input_tensor)); - std::vector& serialized = infer_req->serialized_data_.back(); - - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAppendInputData(irequest, kInputName, serialized.data(), serialized.size(), TRITONSERVER_MEMORY_CPU, 0)); +TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) +{ + ver_out->clear(); + triton::common::TritonJson::Value mv; + if (!slot.Find("model_version", &mv)) { + return nullptr; + } + if (mv.IsString()) { + const char* s; + size_t len; + RETURN_IF_ERR(mv.AsString(&s, &len)); + ver_out->assign(s, len); + return nullptr; + } + if (mv.IsNumber()) { + int64_t iv; + RETURN_IF_ERR(mv.AsInt(&iv)); + *ver_out = std::to_string(iv); + return nullptr; + } + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); +} - RETURN_IF_ERR(TRITONSERVER_InferenceRequestAddRequestedOutput(irequest, kOutputName)); - infer_req->alloc_payload_.output_map_.emplace(std::piecewise_construct, std::forward_as_tuple(kOutputName), std::forward_as_tuple(new HTTPAPIServer::AllocPayload::OutputInfo(HTTPAPIServer::AllocPayload::OutputInfo::JSON, 0))); +#ifdef TRITON_ENABLE_MYSQL_ODBC - return nullptr; +inline double RoundScore6(double x) { + if (!std::isfinite(x)) { + return x; + } + const int64_t scaled = static_cast(std::floor(x * 1e6)); + return static_cast(scaled) / 1e6; } inline uint64_t PackImpCampKey(int imp_idx, int camp_idx) { @@ -155,7 +223,11 @@ bool WriteImpsShapedMultiInferResponse(const std::vector> by_adsize;}; + struct CampAgg { + int32_t cid{0}; + const std::string* mdl{nullptr}; + std::vector> by_adsize; + }; std::unordered_map agg; size_t total_rows = 0; for (size_t si = 0; si < routing_slots.size(); ++si) { @@ -192,7 +264,10 @@ bool WriteImpsShapedMultiInferResponse(const std::vector> camps_per_imp(static_cast(imp_count)); for (const auto& kv : agg) { const int32_t imp = ImpIdxFromPackedKey(kv.first); @@ -252,6 +327,8 @@ bool WriteImpsShapedMultiInferResponse(const std::vector { private: struct FinishPayload { @@ -260,18 +337,34 @@ class MultiInferAggregator : public std::enable_shared_from_this> irequests, - std::vector> imp_routing_slots = {}, + std::vector> irequests +#ifdef TRITON_ENABLE_MYSQL_ODBC + , std::vector> imp_routing_slots = {}, std::vector slot_model_names = {}, - int imp_routing_imp_count = 0) + int imp_routing_imp_count = 0 +#endif + ) : req_(req), n_(slot_count), reply_thread_(reply_thread), - irequests_(std::move(irequests)), success_json_(slot_count), - error_text_(slot_count), have_error_(slot_count, 0), - imp_routing_slots_(std::move(imp_routing_slots)), + irequests_(std::move(irequests)), success_buffers_(slot_count, nullptr), + error_text_(slot_count), have_error_(slot_count, 0) +#ifdef TRITON_ENABLE_MYSQL_ODBC + , imp_routing_slots_(std::move(imp_routing_slots)), slot_model_names_(std::move(slot_model_names)), imp_routing_imp_count_(imp_routing_imp_count) +#endif { +#ifdef TRITON_ENABLE_MYSQL_ODBC slot_row_outputs_.assign(n_, {}); +#endif + } + + ~MultiInferAggregator() + { + for (evbuffer* buf : success_buffers_) { + if (buf != nullptr) { + evbuffer_free(buf); + } + } } std::shared_ptr IrequestAt(size_t i) const @@ -279,6 +372,7 @@ class MultiInferAggregator : public std::enable_shared_from_this 0 && !imp_routing_slots_.empty(); @@ -288,6 +382,7 @@ class MultiInferAggregator : public std::enable_shared_from_this> parsed_first_output = {}, std::vector parsed_scalar_output = {}) { + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, evbuffer* response_json, std::vector> parsed_first_output = {}, std::vector parsed_scalar_output = {}) { if (finalize_err != nullptr) { have_error_[slot] = 1; error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); TRITONSERVER_ErrorDelete(finalize_err); + if (response_json != nullptr) { + evbuffer_free(response_json); + } + CancelAllSubRequests(); } else { - success_json_[slot] = response_json; + success_buffers_[slot] = response_json; +#ifdef TRITON_ENABLE_MYSQL_ODBC if (!parsed_scalar_output.empty() && slot < slot_row_outputs_.size()) { slot_row_outputs_[slot].resize(parsed_scalar_output.size()); for (size_t r = 0; r < parsed_scalar_output.size(); ++r) { - slot_row_outputs_[slot][r] = {static_cast(parsed_scalar_output[r])}; + slot_row_outputs_[slot][r] = { + static_cast(parsed_scalar_output[r]) + }; } } else if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { slot_row_outputs_[slot] = std::move(parsed_first_output); } +#endif } - // Publish per-slot writes before the completion count; the last completer - // schedules WriteHttpReply which reads all slots. std::atomic_thread_fence(std::memory_order_release); const size_t prev = done_count_.fetch_add(1, std::memory_order_acq_rel); - if (prev + 1 < n_) return; + if (prev + 1 < n_) { + return; + } bool expected = false; if (!reply_scheduled_.compare_exchange_strong(expected, true, std::memory_order_acq_rel, std::memory_order_relaxed)) { return; } - std::shared_ptr self = shared_from_this(); - auto* fp = new FinishPayload{std::move(self)}; + auto* fp = new FinishPayload{shared_from_this()}; evthr_defer(reply_thread_, FinishThunk, fp); } @@ -341,9 +443,20 @@ class MultiInferAggregator : public std::enable_shared_from_thisagg->WriteHttpReply(); } + static void AppendShardErrorJson(evbuffer* out, const std::string& message) + { + std::string fragment; + fragment.reserve(message.size() + 24); + fragment += "{\"error\":{\"message\":\""; + AppendJsonEscaped(&fragment, message); + fragment += "\"}}"; + evbuffer_add(out, fragment.data(), fragment.size()); + } + void WriteHttpReply() { std::atomic_thread_fence(std::memory_order_acquire); +#ifdef TRITON_ENABLE_MYSQL_ODBC if (WantsShardParsedRows() && !cancel_sent_.load(std::memory_order_acquire)) { bool any_err = false; for (size_t i = 0; i < n_; ++i) { @@ -368,60 +481,85 @@ class MultiInferAggregator : public std::enable_shared_from_thisbuffer_out, re); - evhtp_send_reply(req_, HttpCodeFromError(re)); - TRITONSERVER_ErrorDelete(re); + if (any_shard_error) { + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value errors(root, triton::common::TritonJson::ValueType::ARRAY); + for (size_t i = 0; i < n_; ++i) { + TRITONSERVER_Error* ae = nullptr; + if (have_error_[i]) { + ae = errors.AppendString(error_text_[i]); + } else { + ae = errors.AppendString(""); + } + if (ae != nullptr) { + LOG_TRITONSERVER_ERROR(ae, "multi_infer: building errors array"); + TRITONSERVER_ErrorDelete(ae); + } + } + TRITONSERVER_Error* re = root.Add("errors", std::move(errors)); + if (re != nullptr) { + LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, re); + evhtp_send_reply(req_, HttpCodeFromError(re)); + TRITONSERVER_ErrorDelete(re); + evhtp_request_resume(req_); + return; + } + triton::common::TritonJson::WriteBuffer wb; + TRITONSERVER_Error* we = root.Write(&wb); + if (we != nullptr) { + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, we); + evhtp_send_reply(req_, HttpCodeFromError(we)); + TRITONSERVER_ErrorDelete(we); + } else { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); + evhtp_send_reply(req_, EVHTP_RES_BADREQ); + } evhtp_request_resume(req_); return; } - triton::common::TritonJson::WriteBuffer wb; - TRITONSERVER_Error* we = root.Write(&wb); - if (we != nullptr) { - AddContentTypeHeader(req_, "application/json"); - EVBufferAddErrorJson(req_->buffer_out, we); - evhtp_send_reply(req_, HttpCodeFromError(we)); - TRITONSERVER_ErrorDelete(we); + } +#endif // TRITON_ENABLE_MYSQL_ODBC + + evbuffer* out = evbuffer_new(); + evbuffer_add(out, "{\"responses\":[", 14); + + for (size_t i = 0; i < n_; ++i) { + if (i > 0) { + evbuffer_add(out, ",", 1); + } + if (have_error_[i]) { + AppendShardErrorJson(out, error_text_[i]); + } else if (success_buffers_[i] == nullptr || evbuffer_get_length(success_buffers_[i]) == 0) { + AppendShardErrorJson(out, "empty multi_infer sub-response"); + if (success_buffers_[i] != nullptr) { + evbuffer_free(success_buffers_[i]); + success_buffers_[i] = nullptr; + } } else { - AddContentTypeHeader(req_, "application/json"); - evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); - evhtp_send_reply(req_, EVHTP_RES_BADREQ); + evbuffer_add_buffer(out, success_buffers_[i]); + evbuffer_free(success_buffers_[i]); + success_buffers_[i] = nullptr; } - evhtp_request_resume(req_); - return; } - static const char kInternalErr[] = "{\"error\":\"unexpected multi_infer success path\"}"; + evbuffer_add(out, "]}", 2); + AddContentTypeHeader(req_, "application/json"); - evbuffer_add(req_->buffer_out, kInternalErr, sizeof(kInternalErr) - 1); - evhtp_send_reply(req_, EVHTP_RES_SERVERR); + evbuffer_add_buffer(req_->buffer_out, out); + evbuffer_free(out); + evhtp_send_reply(req_, EVHTP_RES_OK); evhtp_request_resume(req_); } @@ -431,15 +569,17 @@ class MultiInferAggregator : public std::enable_shared_from_this> irequests_; std::atomic done_count_{0}; - std::vector success_json_; + std::vector success_buffers_; std::vector error_text_; std::vector have_error_; std::atomic cancel_sent_{false}; std::atomic reply_scheduled_{false}; +#ifdef TRITON_ENABLE_MYSQL_ODBC std::vector> imp_routing_slots_; std::vector slot_model_names_; std::vector>> slot_row_outputs_; int imp_routing_imp_count_{0}; +#endif }; class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { @@ -467,6 +607,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); } else if (response != nullptr) { bool skip_shard_json = false; +#ifdef TRITON_ENABLE_MYSQL_ODBC if (infer_request->aggregator_->WantsShardParsedRows()) { const size_t nrows = infer_request->aggregator_->ExpectedRowsForSlot(infer_request->slot_); if (nrows > 0u) { @@ -486,6 +627,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { } } } +#endif if (!skip_shard_json) { shard_json = evbuffer_new(); err = infer_request->FinalizeResponse(response, shard_json); @@ -499,24 +641,13 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); - std::string json_fragment; - if (err == nullptr && shard_json != nullptr) { - const size_t len = evbuffer_get_length(shard_json); - if (len > 0) { - const unsigned char* p = evbuffer_pullup(shard_json, -1); - if (p != nullptr) { - json_fragment.assign(reinterpret_cast(p), len); - } - } - } - if (shard_json != nullptr) { - evbuffer_free(shard_json); - } - if (err != nullptr) { - infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); + if (shard_json != nullptr) { + evbuffer_free(shard_json); + } + infer_request->aggregator_->OnShardDone(infer_request->slot_, err, nullptr); } else { - infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment, std::move(pre_parsed_rows), std::move(pre_parsed_scalars)); + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, shard_json, std::move(pre_parsed_rows), std::move(pre_parsed_scalars)); } if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { @@ -558,28 +689,151 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; - rapidjson::Document parsed; - ImpRoutingTable imp_routing; - std::vector native_slots; + triton::common::TritonJson::Value root; TRITONSERVER_Error* err = nullptr; const size_t body_len = evbuffer_get_length(body_buf); const char* body_ptr = ""; - std::vector body_copy; if (body_len > 0) { - if (const unsigned char* pulled = evbuffer_pullup(body_buf, -1); pulled != nullptr) { - body_ptr = reinterpret_cast(pulled); + const unsigned char* pulled = evbuffer_pullup(body_buf, -1); + if (pulled == nullptr) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); } else { - body_copy.resize(body_len); - const ssize_t nread = evbuffer_copyout(body_buf, body_copy.data(), body_len); - if (nread < 0 || static_cast(nread) != body_len) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); - } else { - body_ptr = body_copy.data(); - } + body_ptr = reinterpret_cast(pulled); } } if (err == nullptr) { - err = triton::server::ParseRequest(body_ptr, body_len, server_.get(), &parsed, &imp_routing, &native_slots); +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (body_len > 0) { + rapidjson::Document imps_doc; + imps_doc.Parse(body_ptr, body_len); + if (!imps_doc.HasParseError() && imps_doc.IsObject() && imps_doc.HasMember("imps") && imps_doc["imps"].IsArray()) { + if (decompressed_buffer != nullptr) { + evbuffer_free(decompressed_buffer); + decompressed_buffer = nullptr; + } + + ImpRoutingTable imp_routing; + std::vector imps_slots; + err = GenerateImpsInferSlots( + imps_doc, server_.get(), &imps_slots, &imp_routing); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + const size_t n = imps_slots.size(); + if (n == 0) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps request produced no inference sub-requests"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + if (n > kMaxMultiInferRequests) { + const std::string lim = "At most " + std::to_string(kMaxMultiInferRequests) + " sub-requests are allowed per multi_infer call"; + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, lim.c_str()); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + for (size_t i = 0; i < n; ++i) { + err = CheckTransactionPolicy(req, imps_slots[i].model_name, imps_slots[i].model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), imps_slots[i].model_name.c_str(), imps_slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused imps multi_infer sub-request"); + } + } + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r), "deleting HTTP imps multi_infer sub-request"); + }); + } + + std::vector slot_model_names; + slot_model_names.reserve(n); + for (const auto& slot : imps_slots) { + slot_model_names.push_back(slot.model_name); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing.slots), std::move(slot_model_names), imp_routing.imp_count); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); + + for (size_t i = 0; i < n; ++i) { + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); + + err = FillImpsTritonRequest(irequests[i].get(), shard.get(), std::move(imps_slots[i])); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } + return; + } + } +#endif // TRITON_ENABLE_MYSQL_ODBC + + err = root.Parse(body_ptr, body_len); } if (decompressed_buffer != nullptr) { evbuffer_free(decompressed_buffer); @@ -594,9 +848,20 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - const size_t n = native_slots.size(); + triton::common::TritonJson::Value requests; + if (!root.Find("requests", &requests)) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + const size_t n = requests.ArraySize(); if (n == 0) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps transform produced no model inference slots"); + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); AddContentTypeHeader(req, "application/json"); EVBufferAddErrorJson(req->buffer_out, err); evhtp_send_reply(req, HttpCodeFromError(err)); @@ -615,23 +880,57 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - std::vector> imp_routing_table; - int imp_routing_imp_count = 0; - if (imp_routing.imp_count > 0 && imp_routing.slots.size() == n) { - imp_routing_table = std::move(imp_routing.slots); - imp_routing_imp_count = imp_routing.imp_count; - } - struct SlotPrep { std::string model_name; - int64_t model_version{-1}; + int64_t model_version{0}; + triton::common::TritonJson::Value infer_json; }; std::vector slots; slots.reserve(n); for (size_t i = 0; i < n; ++i) { + triton::common::TritonJson::Value slot; + err = requests.At(i, &slot); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + const char* mn_c; + size_t mn_len; + err = slot.MemberAsString("model_name", &mn_c, &mn_len); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } SlotPrep prep; - prep.model_name = native_slots[i].model_name; + prep.model_name.assign(mn_c, mn_len); + std::string ver_str; + err = GetModelVersionStringFromSlot(slot, &ver_str); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = GetModelVersionFromString(ver_str, &prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } err = CheckTransactionPolicy(req, prep.model_name, prep.model_version); if (err != nullptr) { AddContentTypeHeader(req, "application/json"); @@ -641,6 +940,17 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } + triton::common::TritonJson::Value infer_only; + err = CopyInferSlotBodyJson(slot, &infer_only); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + prep.infer_json = std::move(infer_only); slots.push_back(std::move(prep)); } @@ -668,13 +978,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { }); } - std::vector slot_model_names; - slot_model_names.reserve(n); - for (const auto& slot : slots) { - slot_model_names.push_back(slot.model_name); - } - - std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing_table),std::move(slot_model_names), imp_routing_imp_count); + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests); std::vector> shard_holders; std::vector> release_holders; shard_holders.reserve(n); @@ -683,7 +987,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { for (size_t i = 0; i < n; ++i) { auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); - err = PopulateInferenceRequestFromNativeSlot(std::move(native_slots[i]), irequests[i].get(), shard.get()); + err = FillMultiInferSlotTritonRequest(slots[i].model_name, slots[i].infer_json, irequests[i].get(), shard.get()); if (err != nullptr) { aggregator->CancelAllSubRequests(); AddContentTypeHeader(req, "application/json"); @@ -694,7 +998,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { return; } - auto rel = std::make_unique(irequests[i], nullptr); + auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); if (err != nullptr) { aggregator->CancelAllSubRequests(); @@ -715,6 +1019,5 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { shard_holders[i].release(); } } -#endif // TRITON_ENABLE_MYSQL_ODBC }} // namespace triton::server diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc index a563dc7629..4114fe33ff 100644 --- a/src/mysql_odbc_connection_pool.cc +++ b/src/mysql_odbc_connection_pool.cc @@ -218,7 +218,7 @@ namespace { constexpr const char kSqlBtModelsMaxTs[] = "SELECT UNIX_TIMESTAMP(MAX(update_timestamp)) FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1"; -constexpr const char kSqlBtModelsForDc[] = "SELECT campaign_id, LOWER(model_name), feature_mapping, feature_sequence, applicable_campaigns FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1 AND dc_id = ?"; +constexpr const char kSqlBtModelsForDc[] = "SELECT campaign_id, model_name, feature_mapping, feature_sequence, applicable_campaigns FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1 AND dc_id = ? ORDER BY update_timestamp DESC"; constexpr size_t kModelNameBuf = kMaxModelNameLen + 1; constexpr size_t kFeatureMappingBuf = kMaxFeatureMappingJsonLen + 1; constexpr size_t kFeatureSequenceBuf = kMaxFeatureSequenceLen + 1; @@ -299,6 +299,22 @@ std::string JsonScalarToString(const rapidjson::Value& v) return {}; } +bool TryParseMappingInt64(const std::string& s, int64_t* out) +{ + if (out == nullptr || s.empty()) { + return false; + } + const char* begin = s.c_str(); + char* end = nullptr; + errno = 0; + const long long v = std::strtoll(begin, &end, 10); + if (errno != 0 || end != begin + static_cast(s.size())) { + return false; + } + *out = static_cast(v); + return true; +} + std::optional FetchMaxUnixTimestampFromDbc(SQLHDBC dbc, const char* sql, int64_t* out_ts) { *out_ts = 0; @@ -394,6 +410,21 @@ int GetFeatureMappingIdx(const char* feature_name, const char* feature, const Fe return inner->second; } +int GetFeatureMappingIdxForInt64(const char* feature_name, int64_t feature_value, const FeatureMappingTables* feature_mapping) { + if (feature_mapping == nullptr || feature_name == nullptr) { + return -1; + } + const auto outer = feature_mapping->find(feature_name); + if (outer == feature_mapping->end()) { + return -1; + } + const auto inner = outer->second.int_value_to_index.find(feature_value); + if (inner == outer->second.int_value_to_index.end()) { + return -1; + } + return inner->second; +} + bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error) { out->clear(); @@ -445,6 +476,10 @@ bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, } table.values.push_back(cell); table.value_to_index[cell] = static_cast(i); + int64_t as_int = 0; + if (TryParseMappingInt64(cell, &as_int)) { + table.int_value_to_index[as_int] = static_cast(i); + } } (*out)[feature_name] = std::move(table); } diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h index 771fcefbfb..55c6fcad78 100644 --- a/src/mysql_odbc_connection_pool.h +++ b/src/mysql_odbc_connection_pool.h @@ -120,6 +120,8 @@ MysqlOdbcConnectionPool* GlobalMysqlOdbcPool(); struct FeatureValueIndexMap { std::vector values; std::unordered_map value_to_index; + // Fast path for JSON numeric features (avoids snprintf per request row). + std::unordered_map int_value_to_index; }; using FeatureMappingTables = std::unordered_map; @@ -131,6 +133,11 @@ int GetFeatureMappingIdx( const char* feature_name, const char* feature, const FeatureMappingTables* feature_mapping); +// Look up categorical index when the request feature value is already numeric. +int GetFeatureMappingIdxForInt64( + const char* feature_name, int64_t feature_value, + const FeatureMappingTables* feature_mapping); + // Loaded from `lightgbm_bt_models` per campaign_id (after merge rules). struct CampaignBtModelBundle { std::string model_name; diff --git a/src/transform.cc b/src/transform.cc index d87e8df9a8..1c39de58ff 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -13,6 +13,7 @@ // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, @@ -29,7 +30,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -52,23 +54,28 @@ int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, if (v.IsString()) { return triton::server::GetFeatureMappingIdx(feature_name, v.GetString(), tables); } - char num_buf[48]; - int n = 0; if (v.IsInt()) { - n = std::snprintf(num_buf, sizeof(num_buf), "%d", v.GetInt()); - } else if (v.IsUint()) { - n = std::snprintf(num_buf, sizeof(num_buf), "%u", v.GetUint()); - } else if (v.IsInt64()) { - n = std::snprintf(num_buf, sizeof(num_buf), "%lld", static_cast(v.GetInt64())); - } else if (v.IsUint64()) { - n = std::snprintf(num_buf, sizeof(num_buf), "%llu", static_cast(v.GetUint64())); - } else { - return -1; + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(v.GetInt()), tables); } - if (n <= 0 || static_cast(n) >= sizeof(num_buf)) { - return -1; + if (v.IsUint()) { + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(v.GetUint()), tables); + } + if (v.IsInt64()) { + return triton::server::GetFeatureMappingIdxForInt64(feature_name, v.GetInt64(), tables); + } + if (v.IsUint64()) { + const uint64_t uv = v.GetUint64(); + if (uv > static_cast(INT64_MAX)) { + char num_buf[32]; + const int n = std::snprintf(num_buf, sizeof(num_buf), "%llu", static_cast(uv)); + if (n <= 0 || static_cast(n) >= sizeof(num_buf)) { + return -1; + } + return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); + } + return triton::server::GetFeatureMappingIdxForInt64(feature_name, static_cast(uv), tables); } - return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); + return -1; } bool IsCampLevelFeature(const std::string& feature) { @@ -158,18 +165,6 @@ TRITONSERVER_Error* FillCampFeaturesInRow(const rapidjson::Value& camp, int32_t return nullptr; } -void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) { - const size_t offset = tensor->size(); - tensor->resize(offset + row.size() * sizeof(float)); - std::memcpy(tensor->data() + offset, row.data(), row.size() * sizeof(float)); -} - -struct ModelSlotBuild { - size_t feature_count{0}; - std::vector tensor; - std::vector routes; -}; - TRITONSERVER_Error* RefreshReadyModelNamesInto(std::unordered_set* out, TRITONSERVER_Server* server) { if (out == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output set pointer is null"); @@ -217,10 +212,54 @@ TRITONSERVER_Error* RefreshReadyModelNamesInto(std::unordered_set* return nullptr; } -} + +} // namespace namespace triton { namespace server { +namespace { + +struct ModelSlotBuild { + size_t feature_count{0}; + std::vector tensor; + std::vector routes; +}; + +void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) +{ + const size_t nbytes = row.size() * sizeof(float); + if (nbytes == 0) { + return; + } + const char* bytes = reinterpret_cast(row.data()); + tensor->insert(tensor->end(), bytes, bytes + nbytes); +} + +TRITONSERVER_Error* CheckModelReadyCached(TRITONSERVER_Server* server, const std::string& model_name, std::unordered_map* ready_cache) { + auto it = ready_cache->find(model_name); + if (it != ready_cache->end()) { + if (!it->second) { + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + } + return nullptr; + } + + bool ready = false; + TRITONSERVER_Error* err = TRITONSERVER_ServerModelIsReady(server, model_name.c_str(), -1 /* latest version */, &ready); + if (err != nullptr) { + return err; + } + (*ready_cache)[model_name] = ready; + if (!ready) { + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + } + return nullptr; +} + +} // namespace + TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server) { if (server == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "server pointer is null"); @@ -243,47 +282,8 @@ const std::unordered_set* ActiveReadyModelNames() { return &g_ready_model_names; } -TRITONSERVER_Error* ParseRequest(const char* json, size_t json_len, TRITONSERVER_Server* server, rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, std::vector* native_slots_out) { - if (out_doc == nullptr) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); - } - - if (imp_routing_out != nullptr) { - imp_routing_out->imp_count = 0; - imp_routing_out->slots.clear(); - } - if (native_slots_out != nullptr) { - native_slots_out->clear(); - } - - rapidjson::Document doc; - doc.Parse(json, json_len); - if (doc.HasParseError()) { - const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); - } - - if (server != nullptr && doc.IsObject() && doc.HasMember(TRITON_BT_JSON_IMPS)) { - const rapidjson::Value& imps_member = doc[TRITON_BT_JSON_IMPS]; - if (imps_member.IsArray()) { - if (native_slots_out == nullptr) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "native_slots_out is required for imps transform"); - } - return GenerateInputVectors(doc, server, out_doc, imp_routing_out, native_slots_out); - } - } - - if (native_slots_out != nullptr) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "multi_infer expects a JSON object with an 'imps' array"); - } - - *out_doc = std::move(doc); - return nullptr; -} - -TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, std::vector* native_slots_out) { - if (out_doc == nullptr || server == nullptr || native_slots_out == nullptr) { +TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITONSERVER_Server* server, std::vector* out_slots, ImpRoutingTable* out_routing) { + if (out_slots == nullptr || server == nullptr) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); } if (!doc.IsObject() || !doc.HasMember(TRITON_BT_JSON_IMPS) || !doc[TRITON_BT_JSON_IMPS].IsArray()) { @@ -295,20 +295,21 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); } - const std::unordered_set* ready_model_names = ActiveReadyModelNames(); - if (ready_model_names == nullptr) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "ready model names are not initialized"); - } - if (ready_model_names->empty()) { - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "no ready models reported by server"); - } - + std::unordered_map ready_cache; std::unordered_map slots_by_model; + const std::string* cached_slot_model_name = nullptr; + ModelSlotBuild* cached_slot_build = nullptr; + TRITONSERVER_Error* err = nullptr; + + out_slots->clear(); + if (out_routing != nullptr) { + out_routing->imp_count = 0; + out_routing->slots.clear(); + } const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; std::vector row; std::unordered_map> imp_base_by_model; - TRITONSERVER_Error* err = nullptr; for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { const rapidjson::Value& imp = imps[ii]; @@ -329,7 +330,7 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON const int32_t campaign_id = camp[TRITON_BT_JSON_CID].GetInt(); auto cmap_it = cmap->find(campaign_id); - if(cmap_it == cmap->end()) { + if (cmap_it == cmap->end()) { cmap_it = cmap->find(0); } if (cmap_it == cmap->end()) { @@ -340,17 +341,28 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON const std::vector& feature_sequence = cmap_it->second.feature_sequence; const FeatureMappingTables& tables = cmap_it->second.feature_mapping; - if (ready_model_names->find(model_name) == ready_model_names->end()) { - const std::string not_ready = "model " + model_name + " not ready"; - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + err = CheckModelReadyCached(server, model_name, &ready_cache); + if (err != nullptr) { + return err; } - ModelSlotBuild& slot = slots_by_model[model_name]; + ModelSlotBuild* slot_build = nullptr; + if (cached_slot_model_name != nullptr && model_name == *cached_slot_model_name) { + slot_build = cached_slot_build; + } else { + auto slot_it = slots_by_model.find(model_name); + if (slot_it == slots_by_model.end()) { + slot_it = slots_by_model.emplace(model_name, ModelSlotBuild{}).first; + } + cached_slot_model_name = &slot_it->first; + cached_slot_build = &slot_it->second; + slot_build = cached_slot_build; + } const size_t feature_count = feature_sequence.size(); - if (slot.feature_count == 0) { - slot.feature_count = feature_count; - slot.tensor.reserve(camps.Size() * feature_count * sizeof(float)); - } else if (slot.feature_count != feature_count) { + if (slot_build->feature_count == 0) { + slot_build->feature_count = feature_count; + } + else if (slot_build->feature_count != feature_count) { return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); } @@ -383,8 +395,10 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { const rapidjson::Value& adsize_item = adsize[ai]; row[static_cast(adsize_idx)] = static_cast(FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables)); - AppendFloatRowToTensor(&slot.tensor, row); - slot.routes.push_back(ImpRouteRow{static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); + AppendFloatRowToTensor(&slot_build->tensor, row); + if (out_routing != nullptr) { + slot_build->routes.push_back(ImpRouteRow{static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); + } } } } @@ -397,11 +411,10 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON } std::sort(model_names.begin(), model_names.end()); - native_slots_out->reserve(model_names.size()); - if (imp_routing_out != nullptr) { - imp_routing_out->imp_count = static_cast(imps.Size()); - imp_routing_out->slots.clear(); - imp_routing_out->slots.reserve(model_names.size()); + out_slots->reserve(model_names.size()); + if (out_routing != nullptr) { + out_routing->imp_count = static_cast(imps.Size()); + out_routing->slots.reserve(model_names.size()); } for (const std::string& model_name : model_names) { @@ -417,19 +430,21 @@ TRITONSERVER_Error* GenerateInputVectors( const rapidjson::Document& doc, TRITON return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); } - MultiInferNativeSlot slot; + ImpsInferSlot slot; slot.model_name = model_name; + slot.model_version = 0; slot.feature_count = built.feature_count; + slot.rows = built.tensor.size() / (built.feature_count * sizeof(float)); slot.input_tensor = std::move(built.tensor); - native_slots_out->push_back(std::move(slot)); + out_slots->push_back(std::move(slot)); - if (imp_routing_out != nullptr) { - imp_routing_out->slots.push_back(std::move(built.routes)); + if (out_routing != nullptr) { + out_routing->slots.push_back(std::move(built.routes)); } } - out_doc->SetObject(); return nullptr; } -} } // namespace triton::server + +}} // namespace triton::server #endif // TRITON_ENABLE_MYSQL_ODBC diff --git a/src/transform.h b/src/transform.h index f3a51e9952..ff1d7ebe4f 100644 --- a/src/transform.h +++ b/src/transform.h @@ -27,7 +27,6 @@ #include #include -#include #include #include @@ -65,24 +64,19 @@ struct ImpRoutingTable { std::vector> slots; }; -// Native FP32 tensor per multi_infer slot (imps transform path; no JSON tensor). -struct MultiInferNativeSlot { +// One model's imps feature matrix ready for direct TRITONSERVER_InferenceRequest +// fill (replaces per-slot infer JSON for the imps fast path). +struct ImpsInferSlot { std::string model_name; - size_t feature_count{0}; - // Row-major FP32 tensor bytes (feature_count * batch_rows * sizeof(float)). + int64_t model_version{0}; + // Row-major FP32 tensor bytes (rows * feature_count * sizeof(float)). std::vector input_tensor; + size_t rows{0}; + size_t feature_count{0}; }; -TRITONSERVER_Error* ParseRequest( - const char* json, size_t json_len, TRITONSERVER_Server* server, - rapidjson::Document* out_doc, - ImpRoutingTable* imp_routing_out = nullptr, - std::vector* native_slots_out = nullptr); - -TRITONSERVER_Error* GenerateInputVectors( - const rapidjson::Document& doc, TRITONSERVER_Server* server, - rapidjson::Document* out_doc, ImpRoutingTable* imp_routing_out, - std::vector* native_slots_out); +constexpr const char* kImpsInputTensorName = "input__0"; +constexpr const char* kImpsOutputTensorName = "output__0"; // Populate the ready-model snapshot once at process startup (after models are loaded). TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server); @@ -90,6 +84,15 @@ TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server); // Lock-free read of the snapshot initialized by InitializeReadyModelNames. const std::unordered_set* ActiveReadyModelNames(); +// Feature mapping + FP32 tensor build for POST /v2/multi_infer imps requests. +TRITONSERVER_Error* GenerateImpsInferSlots( + const rapidjson::Document& doc, TRITONSERVER_Server* server, + std::vector* out_slots, + ImpRoutingTable* out_routing = nullptr); + +// Populates TRITONSERVER_InferenceRequest from a slot: HTTPAPIServer::FillImpsTritonRequest +// in http_server.cc (requires InferRequestClass for input lifetime and output alloc). + #endif // TRITON_ENABLE_MYSQL_ODBC }} // namespace triton::server From b8bf2b2a878c166b0d81ac42d2e150e3e30c138f Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Fri, 10 Jul 2026 16:49:33 +0530 Subject: [PATCH 10/14] Added missing files --- src/multi_infer.cc | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/multi_infer.cc b/src/multi_infer.cc index 302e4e783c..fb59a6c8e0 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -26,8 +26,8 @@ #include "http_server.h" -#include "classification.h" #include "common.h" +#include "http_error_json.h" #ifdef TRITON_ENABLE_MYSQL_ODBC #include "transform.h" #include @@ -39,9 +39,7 @@ #include #include #include -#include #include -#include #include #include #include @@ -77,21 +75,6 @@ int HttpCodeFromError(TRITONSERVER_Error* error) { return EVHTP_RES_BADREQ; } -void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { - triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); - response.AddStringRef("error", message, strlen(message)); - - triton::common::TritonJson::WriteBuffer buffer_json; - response.Write(&buffer_json); - - evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); -} - -void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { - const char* message = TRITONSERVER_ErrorMessage(err); - EVBufferAddErrorJson(buffer, message); -} - void AddContentTypeHeader(evhtp_request_t* req, const char* type) { auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); if (content_header) { From f4e248c43a6edf8bc5b48c9c4b8ebefc261d25be Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Fri, 10 Jul 2026 17:43:49 +0530 Subject: [PATCH 11/14] Added model_verison=-1 for picking latest models --- src/transform.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transform.cc b/src/transform.cc index 1c39de58ff..43e47fc2d5 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -432,7 +432,7 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO ImpsInferSlot slot; slot.model_name = model_name; - slot.model_version = 0; + slot.model_version = -1; slot.feature_count = built.feature_count; slot.rows = built.tensor.size() / (built.feature_count * sizeof(float)); slot.input_tensor = std::move(built.tensor); From 04a63b5135d277e7b6cc7c74b5aafa47805e50e0 Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Mon, 13 Jul 2026 16:19:25 +0530 Subject: [PATCH 12/14] Fixed lower/upper case model names --- src/multi_infer.cc | 2 +- src/mysql_odbc_connection_pool.cc | 14 +++++++++++++- src/mysql_odbc_connection_pool.h | 2 ++ src/transform.cc | 6 +++++- src/transform.h | 1 + 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/multi_infer.cc b/src/multi_infer.cc index fb59a6c8e0..6b95a47489 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -768,7 +768,7 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { std::vector slot_model_names; slot_model_names.reserve(n); for (const auto& slot : imps_slots) { - slot_model_names.push_back(slot.model_name); + slot_model_names.push_back(slot.original_model_name.empty() ? slot.model_name : slot.original_model_name); } std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing.slots), std::move(slot_model_names), imp_routing.imp_count); diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc index 4114fe33ff..81c13f217b 100644 --- a/src/mysql_odbc_connection_pool.cc +++ b/src/mysql_odbc_connection_pool.cc @@ -258,6 +258,16 @@ void Trim(std::string* s) } } +void ToLowerInPlace(std::string* s) +{ + if (s == nullptr) { + return; + } + std::transform(s->begin(), s->end(), s->begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); +} + std::string SqlCharBufferToString(const std::vector& buf, SQLLEN cb) { if (cb == SQL_NULL_DATA) { @@ -503,7 +513,7 @@ namespace { bool TryMergeRowIntoCampaignMap(const LightgbmBtModelRow& row, CampaignToFeatureMappings& out) { - const CampaignBtModelBundle bundle{row.model_name, row.feature_mapping, row.feature_sequence}; + const CampaignBtModelBundle bundle{row.model_name, row.model_name_lower, row.feature_mapping, row.feature_sequence}; if (row.campaign_id != 0) { if (out.find(row.campaign_id) != out.end()) { @@ -628,6 +638,8 @@ std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& if (cb_model_name != SQL_NULL_DATA) { row.model_name = SqlCharBufferToString(model_buf, cb_model_name); Trim(&row.model_name); + row.model_name_lower = row.model_name; + ToLowerInPlace(&row.model_name_lower); } std::string mapping_json; diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h index 55c6fcad78..3d178c7880 100644 --- a/src/mysql_odbc_connection_pool.h +++ b/src/mysql_odbc_connection_pool.h @@ -141,6 +141,7 @@ int GetFeatureMappingIdxForInt64( // Loaded from `lightgbm_bt_models` per campaign_id (after merge rules). struct CampaignBtModelBundle { std::string model_name; + std::string model_name_lower; FeatureMappingTables feature_mapping; std::vector feature_sequence; }; @@ -150,6 +151,7 @@ using CampaignToFeatureMappings = std::unordered_map feature_sequence; std::vector applicable_campaigns; diff --git a/src/transform.cc b/src/transform.cc index 43e47fc2d5..57d972ccc0 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -223,6 +223,7 @@ struct ModelSlotBuild { size_t feature_count{0}; std::vector tensor; std::vector routes; + std::string original_model_name; }; void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) @@ -337,7 +338,8 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO const std::string unknown_campaign = std::string("unknown campaign_id ") + std::to_string(campaign_id); return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, unknown_campaign.c_str()); } - const std::string& model_name = cmap_it->second.model_name; + const std::string& model_name = cmap_it->second.model_name_lower; + const std::string& original_model_name = cmap_it->second.model_name; const std::vector& feature_sequence = cmap_it->second.feature_sequence; const FeatureMappingTables& tables = cmap_it->second.feature_mapping; @@ -353,6 +355,7 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO auto slot_it = slots_by_model.find(model_name); if (slot_it == slots_by_model.end()) { slot_it = slots_by_model.emplace(model_name, ModelSlotBuild{}).first; + slot_it->second.original_model_name = original_model_name; } cached_slot_model_name = &slot_it->first; cached_slot_build = &slot_it->second; @@ -432,6 +435,7 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO ImpsInferSlot slot; slot.model_name = model_name; + slot.original_model_name = built.original_model_name.empty() ? model_name : built.original_model_name; slot.model_version = -1; slot.feature_count = built.feature_count; slot.rows = built.tensor.size() / (built.feature_count * sizeof(float)); diff --git a/src/transform.h b/src/transform.h index ff1d7ebe4f..bc41a672d3 100644 --- a/src/transform.h +++ b/src/transform.h @@ -68,6 +68,7 @@ struct ImpRoutingTable { // fill (replaces per-slot infer JSON for the imps fast path). struct ImpsInferSlot { std::string model_name; + std::string original_model_name; int64_t model_version{0}; // Row-major FP32 tensor bytes (rows * feature_count * sizeof(float)). std::vector input_tensor; From 3875b8b179faffc2346e7d55d218cc1903a67160 Mon Sep 17 00:00:00 2001 From: Shantanu Mirajgave Date: Fri, 24 Jul 2026 14:28:59 +0530 Subject: [PATCH 13/14] Removed odbc.ini file in Dockerfile --- Dockerfile | 3 +-- src/mysql_odbc_connection_pool.cc | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9976e7ceda..32df8c0787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,6 @@ # Run (mount model repo; mount configs directly to /etc): # docker run --name triton1 -d --net=host \ # -v "/tmp/models:/models" \ -# -v "/etc/odbc.ini:/etc/odbc.ini:ro" \ # -v "/etc/triton-dmconfig.json:/etc/triton-dmconfig.json:ro" \ # tritonserver:25.03-custom \ # tritonserver \ @@ -102,4 +101,4 @@ RUN chown 1000:1000 \ ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ && chmod 755 \ ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ - ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ No newline at end of file + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc index 81c13f217b..07ecf29a1b 100644 --- a/src/mysql_odbc_connection_pool.cc +++ b/src/mysql_odbc_connection_pool.cc @@ -54,8 +54,6 @@ std::string BuildMySqlDriverConnectString(const DatabaseConfig& c) std::string driver = c.odbc_driver_name; if (driver.empty()) { driver = "MySQL ODBC 9.7 Unicode Driver"; - } else if (driver == "MySQL ODBC 8.0 Unicode Driver") { - driver = "MySQL ODBC 9.7 Unicode Driver"; } std::ostringstream conn; conn << "DRIVER={" << driver << "};" << "SERVER=" << c.database_ip << ";" << "PORT=" << c.database_port << ";" << "UID={" << c.dsn_user_name << "};" << "PWD={" << c.dsn_user_password << "};"; @@ -741,4 +739,4 @@ const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings() return &g_triton_campaign_feature_mappings[idx]; } -}} // namespace triton::server \ No newline at end of file +}} // namespace triton::server From 3894c13ac4e80e81f88fb9c4c97c464755921aba Mon Sep 17 00:00:00 2001 From: "shantanu.mirajgave" Date: Wed, 5 Aug 2026 04:59:22 -0700 Subject: [PATCH 14/14] Separated predict and multi_infer endpoint. Removed sorting while creating the response. Added minor optimisations --- src/http_server.cc | 6 +- src/http_server.h | 6 +- src/multi_infer.cc | 443 ++++++++++++++---------------- src/mysql_odbc_connection_pool.cc | 12 +- src/mysql_odbc_connection_pool.h | 1 - src/transform.cc | 130 ++++++--- src/transform.h | 2 +- 7 files changed, 322 insertions(+), 278 deletions(-) diff --git a/src/http_server.cc b/src/http_server.cc index bf2d5296db..2803885708 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -4959,11 +4959,15 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingOutput( void HTTPAPIServer::Handle(evhtp_request_t* req) { LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; -#ifdef TRITON_ENABLE_MYSQL_ODBC if (std::string(req->uri->path->full) == "/v2/multi_infer") { HandleMultiInfer(req); return; } +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (std::string(req->uri->path->full) == "/v2/predict") { + HandlePredict(req); + return; + } #endif if (std::string(req->uri->path->full) == "/v2/models/stats") { // model statistics diff --git a/src/http_server.h b/src/http_server.h index 61e0e5b30a..ca1ae55bdc 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -609,7 +609,11 @@ class HTTPAPIServer : public HTTPServer { void HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); - // POST /v2/multi_infer — parallel infer for multiple models in one HTTP call. +#ifdef TRITON_ENABLE_MYSQL_ODBC + // POST /v2/predict — imps-shaped BT inference (feature mapping + model routing). + void HandlePredict(evhtp_request_t* req); +#endif // TRITON_ENABLE_MYSQL_ODBC + // POST /v2/multi_infer — parallel infer for multiple models (requests array). void HandleMultiInfer(evhtp_request_t* req); void HandleModelStats( evhtp_request_t* req, const std::string& model_name = "", diff --git a/src/multi_infer.cc b/src/multi_infer.cc index 6b95a47489..588d65966c 100644 --- a/src/multi_infer.cc +++ b/src/multi_infer.cc @@ -28,6 +28,7 @@ #include "common.h" #include "http_error_json.h" +#include "http_server_macros.h" #ifdef TRITON_ENABLE_MYSQL_ODBC #include "transform.h" #include @@ -42,16 +43,13 @@ #include #include #include +#include #include namespace triton { namespace server { -#include "http_server_macros.h" - namespace { -constexpr size_t kMaxMultiInferRequests = 16; - int HttpCodeFromError(TRITONSERVER_Error* error) { if (error == nullptr) { return EVHTP_RES_OK; @@ -178,27 +176,15 @@ TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Va #ifdef TRITON_ENABLE_MYSQL_ODBC -inline double RoundScore6(double x) { +inline float RoundScore6(float x) { if (!std::isfinite(x)) { return x; } - const int64_t scaled = static_cast(std::floor(x * 1e6)); - return static_cast(scaled) / 1e6; -} - -inline uint64_t PackImpCampKey(int imp_idx, int camp_idx) { - return (static_cast(static_cast(imp_idx)) << 32) | static_cast(static_cast(camp_idx)); + const int64_t scaled = static_cast(std::floor(x * 1e6f)); + return static_cast(scaled) / 1e6f; } -inline int32_t ImpIdxFromPackedKey(uint64_t k) { - return static_cast(static_cast(k >> 32)); -} - -inline int32_t CampIdxFromPackedKey(uint64_t k) { - return static_cast(static_cast(k & 0xffffffffu)); -} - -bool WriteImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector& slot_model_names, std::vector>>& slot_rows, const int imp_count, rapidjson::StringBuffer* sb) { +bool WriteImpsShapedMultiInferResponse(const std::vector>& routing_slots, const std::vector& slot_model_names, const std::vector>& slot_rows, const int imp_count, rapidjson::StringBuffer* sb) { if (imp_count <= 0 || routing_slots.empty() || sb == nullptr) { return false; } @@ -209,15 +195,16 @@ bool WriteImpsShapedMultiInferResponse(const std::vector> by_adsize; + std::vector> by_adsize; + bool initialized{false}; }; - std::unordered_map agg; + size_t total_rows = 0; for (size_t si = 0; si < routing_slots.size(); ++si) { total_rows += routing_slots[si].size(); } - agg.reserve(total_rows); + std::vector> camps_per_imp(static_cast(imp_count)); for (size_t si = 0; si < routing_slots.size(); ++si) { const auto& slot_r = routing_slots[si]; const std::string& slot_mdl = slot_model_names[si]; @@ -225,17 +212,24 @@ bool WriteImpsShapedMultiInferResponse(const std::vector= slot_rows.size()) { return false; } - auto& row_vecs = slot_rows[si]; - if (row_vecs.size() != R) { + const auto& row_scores = slot_rows[si]; + if (row_scores.size() != R) { return false; } for (size_t ri = 0; ri < R; ++ri) { const ImpRouteRow& rc = slot_r[ri]; - const uint64_t pkey = PackImpCampKey(rc.imp_idx, rc.camp_idx); - CampAgg& ca = agg[pkey]; - if (ca.mdl == nullptr) { + if (rc.imp_idx < 0 || rc.imp_idx >= imp_count || rc.camp_idx < 0) { + return false; + } + auto& imp_camps = camps_per_imp[static_cast(rc.imp_idx)]; + if (static_cast(rc.camp_idx) >= imp_camps.size()) { + imp_camps.resize(static_cast(rc.camp_idx) + 1); + } + CampAgg& ca = imp_camps[static_cast(rc.camp_idx)]; + if (!ca.initialized) { ca.cid = rc.cid; ca.mdl = &slot_mdl; + ca.initialized = true; } else if (ca.cid != rc.cid || *ca.mdl != slot_mdl) { return false; } @@ -243,26 +237,9 @@ bool WriteImpsShapedMultiInferResponse(const std::vector= ca.by_adsize.size()) { ca.by_adsize.resize(ad_idx + 1); } - ca.by_adsize[ad_idx] = std::move(row_vecs[ri]); - } - } - - struct CampAggEmitRef { - int32_t camp_idx{0}; - const CampAgg* agg{nullptr}; - }; - std::vector> camps_per_imp(static_cast(imp_count)); - for (const auto& kv : agg) { - const int32_t imp = ImpIdxFromPackedKey(kv.first); - if (imp >= 0 && imp < imp_count) { - camps_per_imp[static_cast(imp)].push_back(CampAggEmitRef{CampIdxFromPackedKey(kv.first), &kv.second}); + ca.by_adsize[ad_idx] = {row_scores[ri]}; } } - for (auto& emits : camps_per_imp) { - std::sort(emits.begin(), emits.end(), [](const CampAggEmitRef& a, const CampAggEmitRef& b) { - return a.camp_idx < b.camp_idx; - }); - } sb->Clear(); sb->Reserve(static_cast(128 + total_rows * 96)); @@ -276,8 +253,10 @@ bool WriteImpsShapedMultiInferResponse(const std::vector(ii)]) { - const CampAgg& ca = *emit.agg; + for (const CampAgg& ca : camps_per_imp[static_cast(ii)]) { + if (!ca.initialized) { + continue; + } writer.StartObject(); writer.Key("cid"); writer.Int(ca.cid); @@ -285,16 +264,16 @@ bool WriteImpsShapedMultiInferResponse(const std::vectorc_str(), static_cast(ca.mdl->size())); writer.Key("score"); writer.StartArray(); - for (const std::vector& vec : ca.by_adsize) { + for (const std::vector& vec : ca.by_adsize) { if (vec.empty()) { continue; } if (vec.size() == 1) { - writer.Double(RoundScore6(vec[0])); + writer.Double(static_cast(RoundScore6(vec[0]))); } else { writer.StartArray(); - for (double d : vec) { - writer.Double(RoundScore6(d)); + for (float f : vec) { + writer.Double(static_cast(RoundScore6(f))); } writer.EndArray(); } @@ -379,7 +358,7 @@ class MultiInferAggregator : public std::enable_shared_from_this> parsed_first_output = {}, std::vector parsed_scalar_output = {}) { + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, evbuffer* response_json, std::vector parsed_scalar_output = {}) { if (finalize_err != nullptr) { have_error_[slot] = 1; error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); @@ -392,14 +371,7 @@ class MultiInferAggregator : public std::enable_shared_from_this(parsed_scalar_output[r]) - }; - } - } else if (!parsed_first_output.empty() && slot < slot_row_outputs_.size()) { - slot_row_outputs_[slot] = std::move(parsed_first_output); + slot_row_outputs_[slot] = std::move(parsed_scalar_output); } #endif } @@ -560,7 +532,7 @@ class MultiInferAggregator : public std::enable_shared_from_this> imp_routing_slots_; std::vector slot_model_names_; - std::vector>> slot_row_outputs_; + std::vector> slot_row_outputs_; int imp_routing_imp_count_{0}; #endif }; @@ -583,7 +555,6 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { TRITONSERVER_Error* err = nullptr; evbuffer* shard_json = nullptr; - std::vector> pre_parsed_rows; std::vector pre_parsed_scalars; if (infer_request->response_count_ != 1) { const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); @@ -598,15 +569,7 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { if (ex_err == nullptr) { skip_shard_json = true; } else { - TRITONSERVER_ErrorDelete(ex_err); - pre_parsed_scalars.clear(); - ex_err = infer_request->ExtractFirstJsonOutputAsRowMajorDoubles(response, nrows, &pre_parsed_rows); - if (ex_err == nullptr) { - skip_shard_json = true; - } else { - TRITONSERVER_ErrorDelete(ex_err); - pre_parsed_rows.clear(); - } + err = ex_err; } } } @@ -624,17 +587,20 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); + if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { + if (shard_json != nullptr) { + evbuffer_free(shard_json); + } + return; + } + if (err != nullptr) { if (shard_json != nullptr) { evbuffer_free(shard_json); } infer_request->aggregator_->OnShardDone(infer_request->slot_, err, nullptr); } else { - infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, shard_json, std::move(pre_parsed_rows), std::move(pre_parsed_scalars)); - } - - if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { - return; + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, shard_json, std::move(pre_parsed_scalars)); } evthr_defer(infer_request->thread_, DeleteMultiInferShardRequestThunk, infer_request); } @@ -648,9 +614,51 @@ class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { const size_t slot_; }; +struct PostBodyContent { + evbuffer* decompressed_buffer{nullptr}; + TRITONSERVER_Error* read_error{nullptr}; + const char* data{""}; + size_t size{0}; +}; + +PostBodyContent ReadPostBody(evhtp_request_t* req, evbuffer* decompressed_buffer) +{ + PostBodyContent out; + out.decompressed_buffer = decompressed_buffer; + evbuffer* body_buf = (out.decompressed_buffer != nullptr) ? out.decompressed_buffer : req->buffer_in; + out.size = evbuffer_get_length(body_buf); + if (out.size > 0) { + const unsigned char* pulled = evbuffer_pullup(body_buf, -1); + if (pulled == nullptr) { + out.read_error = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read request body"); + } else { + out.data = reinterpret_cast(pulled); + } + } + return out; +} + +void FreePostBody(PostBodyContent* body) +{ + if (body->decompressed_buffer != nullptr) { + evbuffer_free(body->decompressed_buffer); + body->decompressed_buffer = nullptr; + } +} + +void RespondWithTritonError(evhtp_request_t* req, TRITONSERVER_Error* err) +{ + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); +} + } // namespace -void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { +#ifdef TRITON_ENABLE_MYSQL_ODBC +void HTTPAPIServer::HandlePredict(evhtp_request_t* req) { RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); if (req->method != htp_method_POST) { @@ -660,174 +668,149 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_pause(req); evbuffer* decompressed_buffer = nullptr; - TRITONSERVER_Error* derr = DecompressBuffer(req, &decompressed_buffer); - if (derr != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, derr); - evhtp_send_reply(req, HttpCodeFromError(derr)); - TRITONSERVER_ErrorDelete(derr); - evhtp_request_resume(req); + TRITONSERVER_Error* read_error = DecompressBuffer(req, &decompressed_buffer); + if (read_error != nullptr) { + RespondWithTritonError(req, read_error); return; } - evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; + PostBodyContent body = ReadPostBody(req, decompressed_buffer); + if (body.read_error != nullptr) { + RespondWithTritonError(req, body.read_error); + FreePostBody(&body); + return; + } - triton::common::TritonJson::Value root; - TRITONSERVER_Error* err = nullptr; - const size_t body_len = evbuffer_get_length(body_buf); - const char* body_ptr = ""; - if (body_len > 0) { - const unsigned char* pulled = evbuffer_pullup(body_buf, -1); - if (pulled == nullptr) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); - } else { - body_ptr = reinterpret_cast(pulled); - } + if (body.size == 0) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array")); + FreePostBody(&body); + return; } - if (err == nullptr) { -#ifdef TRITON_ENABLE_MYSQL_ODBC - if (body_len > 0) { - rapidjson::Document imps_doc; - imps_doc.Parse(body_ptr, body_len); - if (!imps_doc.HasParseError() && imps_doc.IsObject() && imps_doc.HasMember("imps") && imps_doc["imps"].IsArray()) { - if (decompressed_buffer != nullptr) { - evbuffer_free(decompressed_buffer); - decompressed_buffer = nullptr; - } - ImpRoutingTable imp_routing; - std::vector imps_slots; - err = GenerateImpsInferSlots( - imps_doc, server_.get(), &imps_slots, &imp_routing); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + rapidjson::Document imps_doc; + imps_doc.Parse(body.data, body.size); + FreePostBody(&body); - const size_t n = imps_slots.size(); - if (n == 0) { - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps request produced no inference sub-requests"); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - if (n > kMaxMultiInferRequests) { - const std::string lim = "At most " + std::to_string(kMaxMultiInferRequests) + " sub-requests are allowed per multi_infer call"; - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, lim.c_str()); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + if (imps_doc.HasParseError() || !imps_doc.IsObject() || !imps_doc.HasMember("imps") || !imps_doc["imps"].IsArray()) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array")); + return; + } - for (size_t i = 0; i < n; ++i) { - err = CheckTransactionPolicy(req, imps_slots[i].model_name, imps_slots[i].model_version); - if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - } + ImpRoutingTable imp_routing; + std::vector imps_slots; + TRITONSERVER_Error* err = GenerateImpsInferSlots(imps_doc, server_.get(), &imps_slots, &imp_routing); + if (err != nullptr) { + RespondWithTritonError(req, err); + return; + } - evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; - std::vector> irequests; - irequests.reserve(n); - for (size_t i = 0; i < n; ++i) { - TRITONSERVER_InferenceRequest* ireq = nullptr; - err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), imps_slots[i].model_name.c_str(), imps_slots[i].model_version); - if (err != nullptr) { - for (auto& ir : irequests) { - if (ir != nullptr) { - LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused imps multi_infer sub-request"); - } - } - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } - irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { - LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r), "deleting HTTP imps multi_infer sub-request"); - }); - } + const size_t n = imps_slots.size(); + if (n == 0) { + RespondWithTritonError(req, TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "imps request produced no inference sub-requests")); + return; + } - std::vector slot_model_names; - slot_model_names.reserve(n); - for (const auto& slot : imps_slots) { - slot_model_names.push_back(slot.original_model_name.empty() ? slot.model_name : slot.original_model_name); + std::unordered_set policy_checked_models; + for (size_t i = 0; i < n; ++i) { + if (!policy_checked_models.insert(imps_slots[i].model_name).second) { + continue; + } + err = CheckTransactionPolicy(req, imps_slots[i].model_name, imps_slots[i].model_version); + if (err != nullptr) { + RespondWithTritonError(req, err); + return; + } + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), imps_slots[i].model_name.c_str(), imps_slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused predict sub-request"); } + } + RespondWithTritonError(req, err); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r), "deleting HTTP predict sub-request"); + }); + } - std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing.slots), std::move(slot_model_names), imp_routing.imp_count); - std::vector> shard_holders; - std::vector> release_holders; - shard_holders.reserve(n); - release_holders.reserve(n); - - for (size_t i = 0; i < n; ++i) { - auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); - - err = FillImpsTritonRequest(irequests[i].get(), shard.get(), std::move(imps_slots[i])); - if (err != nullptr) { - aggregator->CancelAllSubRequests(); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + std::vector slot_model_names; + slot_model_names.reserve(n); + for (const auto& slot : imps_slots) { + slot_model_names.push_back(slot.original_model_name.empty() ? slot.model_name : slot.original_model_name); + } - auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); - err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); - if (err != nullptr) { - aggregator->CancelAllSubRequests(); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests, std::move(imp_routing.slots), std::move(slot_model_names), imp_routing.imp_count); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); - shard_holders.push_back(std::move(shard)); - release_holders.push_back(std::move(rel)); - } + for (size_t i = 0; i < n; ++i) { + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); - for (size_t i = 0; i < n; ++i) { - release_holders[i].release(); - shard_holders[i].release(); - } - return; - } + err = FillImpsTritonRequest(irequests[i].get(), shard.get(), std::move(imps_slots[i])); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + RespondWithTritonError(req, err); + return; } + + auto rel = std::make_unique(irequests[i], nullptr /* body buffer */); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + RespondWithTritonError(req, err); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } +} #endif // TRITON_ENABLE_MYSQL_ODBC - err = root.Parse(body_ptr, body_len); +void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + + if (req->method != htp_method_POST) { + RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); + } + + evhtp_request_pause(req); + + evbuffer* decompressed_buffer = nullptr; + TRITONSERVER_Error* read_error = DecompressBuffer(req, &decompressed_buffer); + if (read_error != nullptr) { + RespondWithTritonError(req, read_error); + return; } - if (decompressed_buffer != nullptr) { - evbuffer_free(decompressed_buffer); - decompressed_buffer = nullptr; + + PostBodyContent body = ReadPostBody(req, decompressed_buffer); + if (body.read_error != nullptr) { + RespondWithTritonError(req, body.read_error); + FreePostBody(&body); + return; } + + triton::common::TritonJson::Value root; + TRITONSERVER_Error* err = root.Parse(body.data, body.size); + FreePostBody(&body); if (err != nullptr) { - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); + RespondWithTritonError(req, err); return; } @@ -852,16 +835,6 @@ void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { evhtp_request_resume(req); return; } - if (n > kMaxMultiInferRequests) { - const std::string lim = "At most " + std::to_string(kMaxMultiInferRequests) + " sub-requests are allowed per multi_infer call"; - err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, lim.c_str()); - AddContentTypeHeader(req, "application/json"); - EVBufferAddErrorJson(req->buffer_out, err); - evhtp_send_reply(req, HttpCodeFromError(err)); - TRITONSERVER_ErrorDelete(err); - evhtp_request_resume(req); - return; - } struct SlotPrep { std::string model_name; diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc index 07ecf29a1b..5ed9210293 100644 --- a/src/mysql_odbc_connection_pool.cc +++ b/src/mysql_odbc_connection_pool.cc @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include #include #include @@ -49,6 +51,8 @@ namespace { std::atomic g_global_mysql_odbc_pool{nullptr}; +constexpr auto kPoolAcquireTimeout = std::chrono::seconds(30); + std::string BuildMySqlDriverConnectString(const DatabaseConfig& c) { std::string driver = c.odbc_driver_name; @@ -187,7 +191,9 @@ std::optional MysqlOdbcConnectionPool::Initialize() PooledOdbcConnection MysqlOdbcConnectionPool::Acquire() { std::unique_lock lk(mu_); - cv_.wait(lk, [this] { return !free_.empty(); }); + if (!cv_.wait_for(lk, kPoolAcquireTimeout, [this] { return !free_.empty(); })) { + return PooledOdbcConnection(); + } SQLHDBC dbc = free_.front(); free_.pop_front(); return PooledOdbcConnection(this, dbc); @@ -397,6 +403,9 @@ std::optional ParseApplicableCampaignIds(const std::string& s, std: if (endptr == tok.c_str() || *endptr != '\0') { return std::string("invalid campaign id token: ") + tok; } + if (v < INT32_MIN || v > INT32_MAX) { + return std::string("campaign id out of int32_t range: ") + tok; + } out->push_back(static_cast(v)); } return std::nullopt; @@ -729,7 +738,6 @@ std::optional UpdateTritonModelsData() const int inactive = 1 - idx; g_triton_campaign_feature_mappings[inactive] = std::move(by_campaign); g_triton_models_active.store(inactive, std::memory_order_release); - g_triton_campaign_feature_mappings[idx].clear(); return std::nullopt; } diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h index 3d178c7880..0c6044b96c 100644 --- a/src/mysql_odbc_connection_pool.h +++ b/src/mysql_odbc_connection_pool.h @@ -157,7 +157,6 @@ struct LightgbmBtModelRow { std::vector applicable_campaigns; }; -std::optional FetchLightgbmFeatureMappingMaxUpdateUnixSeconds(int64_t* out_ts); std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts); std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map); diff --git a/src/transform.cc b/src/transform.cc index 57d972ccc0..2956570458 100644 --- a/src/transform.cc +++ b/src/transform.cc @@ -221,42 +221,71 @@ namespace { struct ModelSlotBuild { size_t feature_count{0}; + int adsize_idx{-1}; std::vector tensor; std::vector routes; std::string original_model_name; }; -void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) +int AdsizeFeatureIndex(const std::vector& feature_sequence) { - const size_t nbytes = row.size() * sizeof(float); - if (nbytes == 0) { - return; + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + if (feature_sequence[fi] == TRITON_BT_FEATURE_ADSIZE) { + return static_cast(fi); + } } - const char* bytes = reinterpret_cast(row.data()); - tensor->insert(tensor->end(), bytes, bytes + nbytes); + return -1; } -TRITONSERVER_Error* CheckModelReadyCached(TRITONSERVER_Server* server, const std::string& model_name, std::unordered_map* ready_cache) { - auto it = ready_cache->find(model_name); - if (it != ready_cache->end()) { - if (!it->second) { - const std::string not_ready = "model " + model_name + " not ready"; - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); - } +const CampaignBtModelBundle* LookupCampaignBundle(const CampaignToFeatureMappings* cmap, int32_t campaign_id) +{ + auto it = cmap->find(campaign_id); + if (it == cmap->end()) { + it = cmap->find(0); + } + if (it == cmap->end()) { return nullptr; } + return &it->second; +} - bool ready = false; - TRITONSERVER_Error* err = TRITONSERVER_ServerModelIsReady(server, model_name.c_str(), -1 /* latest version */, &ready); - if (err != nullptr) { - return err; +size_t CountCampInferRows(const rapidjson::Value& camp, int adsize_idx) +{ + if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { + return camp[TRITON_BT_FEATURE_ADSIZE].Size(); + } + return 0; +} + +TRITONSERVER_Error* CheckModelReadyFromSnapshot(const std::string& model_name, const std::string& original_model_name, std::unordered_set* verified_models) +{ + if (verified_models->find(model_name) != verified_models->end()) { + return nullptr; } - (*ready_cache)[model_name] = ready; - if (!ready) { - const std::string not_ready = "model " + model_name + " not ready"; - return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + + const std::unordered_set* ready = ActiveReadyModelNames(); + if (ready == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "ready model snapshot is not initialized"); } - return nullptr; + + if (ready->find(model_name) != ready->end() || + (!original_model_name.empty() && ready->find(original_model_name) != ready->end())) { + verified_models->insert(model_name); + return nullptr; + } + + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); +} + +void AppendFloatRowToTensor(std::vector* tensor, const std::vector& row) +{ + const size_t nbytes = row.size() * sizeof(float); + if (nbytes == 0) { + return; + } + const char* bytes = reinterpret_cast(row.data()); + tensor->insert(tensor->end(), bytes, bytes + nbytes); } } // namespace @@ -296,7 +325,33 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); } - std::unordered_map ready_cache; + const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; + + std::unordered_map rows_per_model; + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { + const rapidjson::Value& imp = imps[ii]; + if (!imp.IsObject()) { + continue; + } + if (!imp.HasMember(TRITON_BT_JSON_CAMPS) || !imp[TRITON_BT_JSON_CAMPS].IsArray()) { + continue; + } + const rapidjson::Value& camps = imp[TRITON_BT_JSON_CAMPS]; + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { + const rapidjson::Value& camp = camps[ci]; + if (!camp.IsObject() || !camp.HasMember(TRITON_BT_JSON_CID) || !camp[TRITON_BT_JSON_CID].IsInt()) { + continue; + } + const CampaignBtModelBundle* bundle = LookupCampaignBundle(cmap, camp[TRITON_BT_JSON_CID].GetInt()); + if (bundle == nullptr) { + continue; + } + const int adsize_idx = AdsizeFeatureIndex(bundle->feature_sequence); + rows_per_model[bundle->model_name_lower] += CountCampInferRows(camp, adsize_idx); + } + } + + std::unordered_set verified_ready_models; std::unordered_map slots_by_model; const std::string* cached_slot_model_name = nullptr; ModelSlotBuild* cached_slot_build = nullptr; @@ -308,8 +363,7 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO out_routing->slots.clear(); } - const rapidjson::Value& imps = doc[TRITON_BT_JSON_IMPS]; - std::vector row; + std::vector scratch_row; std::unordered_map> imp_base_by_model; for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { @@ -343,7 +397,7 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO const std::vector& feature_sequence = cmap_it->second.feature_sequence; const FeatureMappingTables& tables = cmap_it->second.feature_mapping; - err = CheckModelReadyCached(server, model_name, &ready_cache); + err = CheckModelReadyFromSnapshot(model_name, original_model_name, &verified_ready_models); if (err != nullptr) { return err; } @@ -356,6 +410,11 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO if (slot_it == slots_by_model.end()) { slot_it = slots_by_model.emplace(model_name, ModelSlotBuild{}).first; slot_it->second.original_model_name = original_model_name; + slot_it->second.adsize_idx = AdsizeFeatureIndex(feature_sequence); + const auto row_est = rows_per_model.find(model_name); + if (row_est != rows_per_model.end() && !feature_sequence.empty()) { + slot_it->second.tensor.reserve(row_est->second * feature_sequence.size() * sizeof(float)); + } } cached_slot_model_name = &slot_it->first; cached_slot_build = &slot_it->second; @@ -379,26 +438,23 @@ TRITONSERVER_Error* GenerateImpsInferSlots(const rapidjson::Document& doc, TRITO base_it = imp_base_by_model.emplace(model_name, std::move(base_row)).first; } - row = base_it->second; - err = FillCampFeaturesInRow(camp, campaign_id, feature_sequence, tables, &row); + if (scratch_row.size() != feature_count) { + scratch_row.resize(feature_count); + } + std::memcpy(scratch_row.data(), base_it->second.data(), feature_count * sizeof(float)); + err = FillCampFeaturesInRow(camp, campaign_id, feature_sequence, tables, &scratch_row); if (err != nullptr) { return err; } - int adsize_idx = -1; - for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { - if (feature_sequence[fi] == TRITON_BT_FEATURE_ADSIZE) { - adsize_idx = static_cast(fi); - break; - } - } + const int adsize_idx = slot_build->adsize_idx; if (adsize_idx >= 0 && camp.HasMember(TRITON_BT_FEATURE_ADSIZE) && camp[TRITON_BT_FEATURE_ADSIZE].IsArray()) { const rapidjson::Value& adsize = camp[TRITON_BT_FEATURE_ADSIZE]; for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { const rapidjson::Value& adsize_item = adsize[ai]; - row[static_cast(adsize_idx)] = static_cast(FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables)); - AppendFloatRowToTensor(&slot_build->tensor, row); + scratch_row[static_cast(adsize_idx)] = static_cast(FeatureIdxFromJsonValue(TRITON_BT_FEATURE_ADSIZE, adsize_item, &tables)); + AppendFloatRowToTensor(&slot_build->tensor, scratch_row); if (out_routing != nullptr) { slot_build->routes.push_back(ImpRouteRow{static_cast(ii), static_cast(ci), static_cast(ai), campaign_id}); } diff --git a/src/transform.h b/src/transform.h index bc41a672d3..c11dafd095 100644 --- a/src/transform.h +++ b/src/transform.h @@ -85,7 +85,7 @@ TRITONSERVER_Error* InitializeReadyModelNames(TRITONSERVER_Server* server); // Lock-free read of the snapshot initialized by InitializeReadyModelNames. const std::unordered_set* ActiveReadyModelNames(); -// Feature mapping + FP32 tensor build for POST /v2/multi_infer imps requests. +// Feature mapping + FP32 tensor build for POST /v2/predict imps requests. TRITONSERVER_Error* GenerateImpsInferSlots( const rapidjson::Document& doc, TRITONSERVER_Server* server, std::vector* out_slots,