diff --git a/.gitignore b/.gitignore index 31f5d9f5f4..01abc6eaab 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ qa/L0_openai/openai tensorrtllm_models tensorrtllm_mistral_models/ custom_tokenizer +replace-artifacts/ \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index bd91a52109..de1e229c2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,7 +24,22 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required(VERSION 3.31.8) +cmake_minimum_required(VERSION 3.18) + +# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5). +# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a +# separate CMake invocation; that step does not inherit -D variables from the top +# level unless they are also in the environment. If configure fails inside libevent +# with "Compatibility with CMake < 3.5 has been removed", use either: +# export CMAKE_POLICY_VERSION_MINIMUM=3.5 # before cmake AND cmake --build +# or install/use CMake 3.28.x–3.31.x for this build. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + message( + STATUS + "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export " + "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build " + "(see docs/customization_guide/build.md).") +endif() project(tritonserver LANGUAGES C CXX) @@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) +option(TRITON_ENABLE_MYSQL_ODBC + "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)" + OFF) + # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) @@ -166,9 +185,22 @@ if (OPENSSL_ROOT_DIR) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${OPENSSL_ROOT_DIR}") endif() -set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") +# Location where protobuf-config.cmake will be installed varies by +# platform +if (WIN32) + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/cmake") +else() + set(_FINDPACKAGE_PROTOBUF_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/protobuf/${LIB_DIR}/cmake/protobuf") +endif() -set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp") +# Triton with Opentelemetry is not supported on Windows +# FIXME: add location for Windows, when support is added +# JIRA DLIS-4786 +if (WIN32) + set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "") +else() + set(_FINDPACKAGE_OPENTELEMETRY_CONFIG_DIR "${TRITON_THIRD_PARTY_INSTALL_PREFIX}/opentelemetry-cpp/${LIB_DIR}/cmake/opentelemetry-cpp") +endif() if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(TRITON_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/install) @@ -189,7 +221,7 @@ endif() # TRITON_ENABLE_HTTP || TRITON_ENABLE_METRICS || TRITON_ENABLE_SAGEMAKER if(${TRITON_ENABLE_GRPC}) set(TRITON_DEPENDS ${TRITON_DEPENDS} grpc) endif() # TRITON_ENABLE_GRPC -if(${TRITON_ENABLE_TRACING}) +if(NOT WIN32 AND ${TRITON_ENABLE_TRACING}) set(TRITON_DEPENDS ${TRITON_DEPENDS} opentelemetry-cpp) endif() # TRITON_ENABLE_TRACING @@ -248,6 +280,7 @@ ExternalProject_Add(triton-server -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} + -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0d01b1a996..6b9d3378dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -88,7 +88,7 @@ proposed change so that the Triton team can provide feedback. documentation for instructions on running these tests. - Triton Inference Server's default build assumes recent versions of - dependencies (CUDA, PyTorch, TensorRT, + dependencies (CUDA, TensorFlow, PyTorch, TensorRT, etc.). Contributions that add compatibility with older versions of those dependencies will be considered, but NVIDIA cannot guarantee that all possible build configurations work, are not broken by diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..27454c0d38 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Derivative image: start from an official Triton server image and replace only +# the tritonserver executable and libtritonserver.so. Backends, Python wheels, +# and the rest of the filesystem stay unchanged from the base image. +# +# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX): +# install/bin/tritonserver +# install/lib/libtritonserver.so +# +# Prepare artifacts (from repository root): +# mkdir -p replace-artifacts +# cp install/bin/tritonserver replace-artifacts/ +# cp install/lib/libtritonserver.so replace-artifacts/ +# cp /etc/odbc.ini replace-artifacts/odbc.ini # optional if databaseIp is set (DSN unused) +# Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for +# optional JSON field "odbcDriverName" if the default fails. +# +# Build (from repository root; match your Triton tag, e.g. r25.03): +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \ +# -t tritonserver:25.03-custom . +# +# CPU-only base example: +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \ +# -t tritonserver:25.03-custom-cpu . +# +# If your base image stores libtritonserver.so under lib64: +# --build-arg TRITON_LIB_SUBDIR=lib64 +# +# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.: +# --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04 +# +# Run (mount a model repository and optionally override DM config): +# docker run --rm --gpus=all \ +# -p8000:8000 -p8001:8001 -p8002:8002 \ +# -v /path/to/model_repo:/models:ro \ +# -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \ +# tritonserver:25.03-custom \ +# tritonserver --model-repository=/models +# +# For CPU-only, drop --gpus=all and use a CPU/min base image. + +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 + +FROM ${BASE_IMAGE} + +ARG TRITON_INSTALL_PREFIX=/opt/tritonserver +ARG TRITON_LIB_SUBDIR=lib + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe +# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc. +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in +# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under +# /usr/lib//odbc/ only. unixODBC then fails with "Can't open lib +# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/. +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json). +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro) +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json + +# Paths relative to the build context (repository root when building with ".") +COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver +COPY replace-artifacts/libtritonserver.so \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + +# Match ownership used by generated Triton Dockerfiles (triton-server uid) +RUN chown 1000:1000 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ + && chmod 755 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + diff --git a/Dockerfile.QA b/Dockerfile.QA index caa5fb0e2c..7406cf2345 100644 --- a/Dockerfile.QA +++ b/Dockerfile.QA @@ -1,4 +1,4 @@ -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -59,23 +59,35 @@ RUN apt-get update && \ libboost-dev \ python3-dev \ python3-pip \ + python3-wheel \ python3-setuptools \ python3-venv \ - python3-wheel \ rapidjson-dev \ software-properties-common && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install cmake==4.0.3 -ENV CMAKE_POLICY_VERSION_MINIMUM=3.5 +RUN apt update -q=2 \ + && apt install -y gpg wget \ + && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ + && . /etc/os-release \ + && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ + && apt-get update -q=2 \ + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* -# Add densenet_onnx model to example repo +# Add inception_graphdef model to example repo # FIXME: This should be changed to using the fetch_models.sh script # in order to ensure the public facing docs are up-to-date. WORKDIR /workspace/docs/examples/model_repository -RUN mkdir -p densenet_onnx/1 && \ - wget -O densenet_onnx/1/model.onnx \ - https://github.com/onnx/models/raw/main/validated/vision/classification/densenet-121/model/densenet-7.onnx +RUN mkdir -p model_repository/inception_onnx/1 && \ + wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \ + https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \ + (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \ + python3 -m venv tf2onnx && \ + source ./tf2onnx/bin/activate && \ + pip3 install "numpy<2" tensorflow tf2onnx && \ + python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 && \ + deactivate && \ + mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx # Update the qa/ directory with test executables, models, etc. WORKDIR /workspace @@ -105,7 +117,7 @@ RUN mkdir -p qa/common && \ cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \ - cp -r docs/examples/model_repository/densenet_onnx qa/L0_grpc/models && \ + cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \ mkdir qa/L0_grpc_state_cleanup/models && \ cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \ mkdir qa/L0_http/models && \ @@ -114,7 +126,7 @@ RUN mkdir -p qa/common && \ cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \ - cp -r docs/examples/model_repository/densenet_onnx qa/L0_http/models && \ + cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \ mkdir qa/L0_https/models && \ cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \ mkdir qa/L0_secure_grpc/models && \ @@ -139,7 +151,6 @@ RUN mkdir -p qa/common && \ mkdir qa/L0_data_compression/models && \ cp -r docs/examples/model_repository/simple qa/L0_data_compression/models && \ cp bin/data_compressor_test qa/L0_data_compression/. && \ - cp bin/tensor_size_test qa/L0_input_validation/. && \ cp bin/metrics_api_test qa/L0_metrics/. && \ cp bin/response_cache_test qa/L0_response_cache/. && \ cp bin/request_cancellation_test qa/L0_request_cancellation/. && \ @@ -174,8 +185,8 @@ RUN mkdir -p qa/custom_models/custom_sequence_int32/1 && \ qa/custom_models/custom_dyna_sequence_int32/1/. # L0_lifecycle needs No-GPU build of identity backend. -WORKDIR /workspace/tritonbuild/identity -RUN rm -rf install build && mkdir build && cd build && \ +RUN cd tritonbuild/identity && \ + rm -rf install build && mkdir build && cd build && \ cmake -DTRITON_ENABLE_GPU=OFF \ -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/identity/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ @@ -186,18 +197,15 @@ RUN rm -rf install build && mkdir build && cd build && \ make -j16 install # L0_backend_python test require triton_shm_monitor -ARG TRITON_BOOST_URL="https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz" -WORKDIR /workspace/tritonbuild/python -RUN rm -rf install build && mkdir build && cd build && \ +RUN cd tritonbuild/python && \ + rm -rf install build && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/python/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \ - -DTRITON_BOOST_URL:STRING=${TRITON_BOOST_URL} \ -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \ make -j16 triton-shm-monitor install -WORKDIR /workspace/ RUN cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \ qa/L0_lifecycle/. && \ cp tritonbuild/python/install/backends/python/triton_shm_monitor*.so \ @@ -317,9 +325,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN if grep -qE '^VERSION_ID="(18\.04|20\.04|22\.04|24\.04)' /etc/os-release; then \ apt-get update && \ apt-get install -y --no-install-recommends \ - libpng-dev && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; \ + libpng-dev; \ else \ echo "Ubuntu version must be either 18.04, 20.04, 22.04 or 24.04" && \ exit 1; \ @@ -330,23 +336,22 @@ RUN if grep -qE '^VERSION_ID="(18\.04|20\.04|22\.04|24\.04)' /etc/os-release; th RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ gdb \ + libopencv-dev \ libarchive-dev \ libopencv-core-dev \ - libopencv-dev \ libzmq3-dev \ + openjdk-11-jdk \ nginx \ npm \ - openjdk-11-jdk \ protobuf-compiler \ python3-dev \ python3-pip \ python3-protobuf \ - python3-setuptools \ python3-wheel \ + python3-setuptools \ swig \ valgrind && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + rm -rf /var/lib/apt/lists/* # CI/QA expects "python" executable (not python3). RUN rm -f /usr/bin/python && \ @@ -378,17 +383,10 @@ COPY --chown=1000:1000 --from=sdk /workspace/qa/ qa/ # Remove CI tests that are meant to run only on build image and # install the tritonserver/triton python client APIs. -RUN rm -fr qa/L0_copyrights qa/L0_build_variants - - -RUN --mount=type=secret,id=triton_ci_pip_extra_values,env=TRITON_CI_PYPI_EXTRA_VALUES \ - if [ -n "${TRITON_CI_PYPI_EXTRA_VALUES}" ]; then \ - find qa/pkgs/ -maxdepth 1 -type f -name \ - "tritonclient-*any*.whl" -exec pip3 install --upgrade ${TRITON_CI_PYPI_EXTRA_VALUES} {}[all] \; ; \ - else \ - find qa/pkgs/ -maxdepth 1 -type f -name \ - "tritonclient-*any*.whl" -exec pip3 install --upgrade {}[all] \; ; \ - fi +RUN rm -fr qa/L0_copyrights qa/L0_build_variants && \ + find qa/pkgs/ -maxdepth 1 -type f -name \ + "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ + xargs pip3 install --upgrade ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH} diff --git a/Dockerfile.pubmatic-bt-tritonserver b/Dockerfile.pubmatic-bt-tritonserver new file mode 100644 index 0000000000..340c3b27b5 --- /dev/null +++ b/Dockerfile.pubmatic-bt-tritonserver @@ -0,0 +1,52 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Extend the image produced by build.py (`docker build -t tritonserver`) with +# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files. +# Build context: repository root (same as build.py final docker build). +# +# Prerequisites: +# - Image `tritonserver` must exist. +# - replace-artifacts/odbc.ini +# - replace-artifacts/triton-dmconfig.json +# +# Example: +# docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver . + +FROM tritonserver + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json diff --git a/Dockerfile.sdk b/Dockerfile.sdk index b2181abe6e..de6b12df5e 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -1,4 +1,4 @@ -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,19 +29,22 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:26.05-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo +ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server ARG TRITON_COMMON_REPO_TAG=main ARG TRITON_CORE_REPO_TAG=main ARG TRITON_CLIENT_REPO_TAG=main ARG TRITON_THIRD_PARTY_REPO_TAG=main +ARG TRITON_MODEL_ANALYZER_REPO_TAG=main ARG TRITON_ENABLE_GPU=ON ARG JAVA_BINDINGS_MAVEN_VERSION=3.8.4 ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG=1.5.8 + # DCGM version to install for Model Analyzer -ARG DCGM_VERSION=4.5.3-1 +ARG DCGM_VERSION=3.3.6 ARG NVIDIA_TRITON_SERVER_SDK_VERSION=unknown ARG NVIDIA_BUILD_ID=unknown @@ -54,39 +57,48 @@ FROM ${BASE_IMAGE} AS sdk_build # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive -ENV PIP_BREAK_SYSTEM_PACKAGES=1 CMAKE_POLICY_VERSION_MINIMUM=3.5 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 RUN apt-get update && \ apt-get install -y --no-install-recommends \ + ca-certificates \ + software-properties-common \ autoconf \ automake \ build-essential \ - ca-certificates \ curl \ git \ gperf \ libb64-dev \ libgoogle-perftools-dev \ - libopencv-core-dev \ libopencv-dev \ + libopencv-core-dev \ libssl-dev \ libtool \ - maven \ - openjdk-11-jdk \ pkg-config \ python3 \ - python3-dev \ - python3-pdfkit \ python3-pip \ - python3-setuptools \ + python3-dev \ python3-wheel \ + python3-setuptools \ rapidjson-dev \ - software-properties-common \ vim \ - wget && \ - pip3 install --upgrade "grpcio-tools<1.68" cmake==4.0.3 auditwheel - -ENV CMAKE_POLICY_MINIMUM_REQUIRED=3.5 + wget \ + python3-pdfkit \ + openjdk-11-jdk \ + maven && \ + pip3 install --upgrade "grpcio-tools<1.68" + +# Client build requires recent version of CMake (FetchContent required) +# Using CMAKE installation instruction from:: https://apt.kitware.com/ +RUN apt update -q=2 \ + && apt install -y gpg wget \ + && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ + && . /etc/os-release \ + && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ + && apt-get update -q=2 \ + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* \ + && cmake --version # Build expects "python" executable (not python3). RUN rm -f /usr/bin/python && \ @@ -95,6 +107,7 @@ RUN rm -f /usr/bin/python && \ # Build the client library and examples ARG TRITON_REPO_ORGANIZATION ARG TRITON_CLIENT_REPO_SUBDIR +ARG TRITON_PA_REPO_SUBDIR ARG TRITON_COMMON_REPO_TAG ARG TRITON_CORE_REPO_TAG ARG TRITON_CLIENT_REPO_TAG @@ -107,6 +120,7 @@ ARG TARGETPLATFORM WORKDIR /workspace COPY TRITON_VERSION . COPY ${TRITON_CLIENT_REPO_SUBDIR} client +COPY ${TRITON_PA_REPO_SUBDIR} perf_analyzer WORKDIR /workspace/client_build RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ @@ -117,11 +131,42 @@ RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_THIRD_PARTY_REPO_TAG=${TRITON_THIRD_PARTY_REPO_TAG} \ -DTRITON_ENABLE_PERF_ANALYZER=OFF \ -DTRITON_ENABLE_CC_HTTP=ON -DTRITON_ENABLE_CC_GRPC=ON \ - -DTRITON_ENABLE_PYTHON_HTTP=ON -DTRITON_ENABLE_PYTHON_GRPC=ON \ + -DTRITON_ENABLE_PYTHON_HTTP=OFF -DTRITON_ENABLE_PYTHON_GRPC=OFF \ -DTRITON_ENABLE_JAVA_HTTP=ON \ -DTRITON_ENABLE_EXAMPLES=ON -DTRITON_ENABLE_TESTS=ON \ -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} /workspace/client -RUN cmake --build . -v --parallel --target cc-clients java-clients python-clients +RUN make -j16 cc-clients java-clients && \ + rm -fr ~/.m2 + +# TODO: PA will rebuild the CC clients since it depends on it. +# This should be optimized so that we do not have to build +# the CC clients twice. Similarly, because the SDK expectation is +# that PA is packaged with the python client, we hold off on building +# the python client until now. Post-migration we should focus +# effort on de-tangling these flows. +WORKDIR /workspace/pa_build +RUN cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ + -DTRITON_VERSION=`cat /workspace/TRITON_VERSION` \ + -DTRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ + -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ + -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ + -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_OPENAI=ON \ + -DTRITON_ENABLE_CC_HTTP=ON \ + -DTRITON_ENABLE_CC_GRPC=ON \ + -DTRITON_ENABLE_PYTHON_HTTP=ON \ + -DTRITON_ENABLE_PYTHON_GRPC=ON \ + -DTRITON_PACKAGE_PERF_ANALYZER=ON \ + -DTRITON_ENABLE_GPU=${TRITON_ENABLE_GPU} \ + /workspace/perf_analyzer +RUN make -j16 perf-analyzer python-clients + +RUN pip3 install build \ + && cd /workspace/perf_analyzer/genai-perf \ + && python3 -m build --wheel --outdir /workspace/install/python # Install Java API Bindings RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \ @@ -149,40 +194,37 @@ ARG TRITON_ENABLE_GPU RUN apt-get update && \ apt-get install -y --no-install-recommends \ + software-properties-common \ curl \ - default-jdk \ git \ gperf \ libb64-dev \ libgoogle-perftools-dev \ - libopencv-core-dev \ libopencv-dev \ + libopencv-core-dev \ libssl-dev \ libtool \ - maven \ - perl \ python3 \ - python3-dev \ - python3-pdfkit \ python3-pip \ - python3-setuptools \ + python3-dev \ python3-wheel \ + python3-setuptools \ vim \ - wget && \ - pip3 install "grpcio<1.68" "grpcio-tools<1.68" && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*; + wget \ + python3-pdfkit \ + maven \ + default-jdk && \ + pip3 install "grpcio<1.68" "grpcio-tools<1.68" WORKDIR /workspace COPY TRITON_VERSION . COPY NVIDIA_Deep_Learning_Container_License.pdf . COPY --from=sdk_build /workspace/client/ client/ +COPY --from=sdk_build /workspace/perf_analyzer/ perf_analyzer/ COPY --from=sdk_build /workspace/install/ install/ -WORKDIR /workspace/install -RUN export VERSION=`cat /workspace/TRITON_VERSION` && \ - tar zcf /workspace/v"$VERSION".clients.tar.gz ./* - -WORKDIR /workspace +RUN cd install && \ + export VERSION=`cat /workspace/TRITON_VERSION` && \ + tar zcf /workspace/v$VERSION.clients.tar.gz * # For CI testing need to copy over L0_sdk test and L0_client_build_variants test. RUN mkdir qa @@ -196,38 +238,23 @@ COPY --from=sdk_build /workspace/client/src/python/library/tests/* qa/python_cli # Install an image needed by the quickstart and other documentation. COPY qa/images/mug.jpg images/mug.jpg +RUN pip3 install install/python/genai_perf-*.whl + # Install the dependencies needed to run the client examples. These # are not needed for building but including them allows this image to # be used to run the client examples. -RUN pip3 install --upgrade "numpy<2" pillow attrdict - -RUN --mount=type=secret,id=triton_ci_pip_extra_values,env=TRITON_CI_PYPI_EXTRA_VALUES \ - if [ -n "${TRITON_CI_PYPI_EXTRA_VALUES}" ]; then \ - find install/python/ -maxdepth 1 -type f -name \ - "tritonclient-*any*.whl" -exec pip3 install --upgrade ${TRITON_CI_PYPI_EXTRA_VALUES} {}[all] \; ; \ - else \ - find install/python/ -maxdepth 1 -type f -name \ - "tritonclient-*any*.whl" -exec pip3 install --upgrade {}[all] \; ; \ - fi - -# Install GenAI-Perf -RUN --mount=type=secret,id=triton_ci_pip_extra_values,env=TRITON_CI_PYPI_EXTRA_VALUES \ - if [ -n "${TRITON_CI_PYPI_EXTRA_VALUES}" ]; then \ - pip3 install --upgrade ${TRITON_CI_PYPI_EXTRA_VALUES} genai-perf ; \ - else \ - pip3 install --upgrade genai-perf ; \ - fi +RUN pip3 install --upgrade "numpy<2" pillow attrdict && \ + find install/python/ -maxdepth 1 -type f -name \ + "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ + xargs pip3 install --upgrade # Install DCGM RUN if [ "$TRITON_ENABLE_GPU" = "ON" ]; then \ [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" && \ curl -o /tmp/cuda-keyring.deb \ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/${arch}/cuda-keyring_1.1-1_all.deb" \ - && apt -y install /tmp/cuda-keyring.deb && rm /tmp/cuda-keyring.deb && \ - apt update && \ - apt install --yes --no-install-recommends \ - datacenter-gpu-manager-4-core=1:${DCGM_VERSION} \ - datacenter-gpu-manager-4-dev=1:${DCGM_VERSION}; \ + https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/$arch/cuda-keyring_1.1-1_all.deb \ + && apt install /tmp/cuda-keyring.deb && rm /tmp/cuda-keyring.deb && \ + apt-get update && apt-get install -y datacenter-gpu-manager=1:${DCGM_VERSION}; \ fi # Build expects "python" executable (not python3). diff --git a/Dockerfile.win10.min b/Dockerfile.win10.min new file mode 100644 index 0000000000..b883ad4dcb --- /dev/null +++ b/Dockerfile.win10.min @@ -0,0 +1,201 @@ +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Windows min container for Triton build + +ARG BASE_IMAGE=mcr.microsoft.com/windows:10.0.19042.1889 + +FROM ${BASE_IMAGE} as dependency_base + +RUN powershell.exe Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope LocalMachine +RUN powershell.exe [Net.ServicePointManager]::Expect100Continue=$true;[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls,[Net.SecurityProtocolType]::Tls11,[Net.SecurityProtocolType]::Tls12,[Net.SecurityProtocolType]::Ssl3;Invoke-Expression( New-Object System.Net.WebClient ).DownloadString('https://chocolatey.org/install.ps1') +RUN choco install unzip -y + +# +# Installing TensorRT +# +ARG TENSORRT_VERSION=10.8.0.43 +ARG TENSORRT_ZIP="TensorRT-${TENSORRT_VERSION}.Windows.win10.cuda-12.8.zip" +ARG TENSORRT_SOURCE=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-${TENSORRT_VERSION}.Windows.win10.cuda-12.8.zip +# COPY ${TENSORRT_ZIP} /tmp/${TENSORRT_ZIP} +ADD ${TENSORRT_SOURCE} /tmp/${TENSORRT_ZIP} +RUN unzip /tmp/%TENSORRT_ZIP% +RUN move TensorRT-* TensorRT + +LABEL TENSORRT_VERSION="${TENSORRT_VERSION}" + + +# +# Installing cuDNN +# +ARG CUDNN_VERSION=9.7.1.26 +ARG CUDNN_ZIP=cudnn-windows-x86_64-${CUDNN_VERSION}_cuda12-archive.zip +ARG CUDNN_SOURCE=https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/windows-x86_64/cudnn-windows-x86_64-9.7.1.26_cuda12-archive.zip +ADD ${CUDNN_SOURCE} /tmp/${CUDNN_ZIP} +RUN unzip /tmp/%CUDNN_ZIP% +RUN move cudnn-* cudnn + +LABEL CUDNN_VERSION="${CUDNN_VERSION}" + + +FROM ${BASE_IMAGE} as build_base + +SHELL ["cmd", "/S", "/C"] + +RUN mkdir c:\tmp +WORKDIR /tmp + +RUN powershell.exe Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope LocalMachine +RUN powershell.exe [Net.ServicePointManager]::Expect100Continue=$true;[Net.ServicePointManager]::SecurityProtocol=[Net.SecurityProtocolType]::Tls,[Net.SecurityProtocolType]::Tls11,[Net.SecurityProtocolType]::Tls12,[Net.SecurityProtocolType]::Ssl3;Invoke-Expression( New-Object System.Net.WebClient ).DownloadString('https://chocolatey.org/install.ps1') +RUN choco install git docker unzip -y + +# +# Installing python +# +ARG PYTHON_VERSION=3.12.3 +ARG PYTHON_SOURCE=https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-amd64.exe +ADD ${PYTHON_SOURCE} python-${PYTHON_VERSION}-amd64.exe +RUN python-%PYTHON_VERSION%-amd64.exe /quiet InstallAllUsers=1 PrependPath=1 Include_doc=0 TargetDir="C:\python%PYTHON_VERSION%" +RUN mklink "C:\python%PYTHON_VERSION%\python3.exe" "C:\python%PYTHON_VERSION%\python.exe" +RUN pip install --upgrade wheel setuptools docker + +LABEL PYTHON_VERSION=${PYTHON_VERSION} + +# +# Installing CMake +# +ARG CMAKE_VERSION=3.30.5 +RUN pip install cmake==%CMAKE_VERSION% + +ENV CMAKE_TOOLCHAIN_FILE /vcpkg/scripts/buildsystems/vcpkg.cmake +ENV VCPKG_TARGET_TRIPLET x64-windows + +LABEL CMAKE_VERSION=${CMAKE_VERSION} + +# Be aware that pip can interact badly with VS cmd shell so need to pip install before +# vsdevcmd.bat (see https://bugs.python.org/issue38989) +# +# Installing Visual Studio BuildTools: VS17 2022 +# +# Download collect.exe in case of an install failure. +ADD https://aka.ms/vscollect.exe "C:\tmp\collect.exe" + +# Use the latest release channel. For more control, specify the location of an internal layout. +# Download the Build Tools bootstrapper. +# ARG BUILD_TOOLS_SOURCE=https://aka.ms/vs/17/release/vs_buildtools.exe + +ARG BUILDTOOLS_VERSION=17.12.35506.116 +ARG BUILD_TOOLS_SOURCE=https://download.visualstudio.microsoft.com/download/pr/5536698c-711c-4834-876f-2817d31a2ef2/58894fc272e86d3c3a6d85bf3a1df1e5a0685be8b9ab65d9f3cc5c2a8c6921cc/vs_BuildTools.exe + +ADD ${BUILD_TOOLS_SOURCE} vs_buildtools.exe +# Install Build Tools with the Microsoft.VisualStudio.Workload.VCTools workload, including recommended. +ARG VS_INSTALL_PATH_WP="C:\BuildTools" +RUN vs_buildtools.exe --quiet --wait --norestart --nocache install \ + --installPath %VS_INSTALL_PATH_WP% \ + --add Microsoft.VisualStudio.Workload.VCTools \ + --includeRecommended \ + --locale "En-us" + +LABEL BUILDTOOLS_VERSION=${BUILDTOOLS_VERSION} + +WORKDIR / + +# +# Installing Vcpkg +# +ARG VCPGK_VERSION=2024.03.19 +RUN git clone --single-branch --depth=1 -b %VCPGK_VERSION% https://github.com/microsoft/vcpkg.git +WORKDIR /vcpkg +RUN bootstrap-vcpkg.bat +RUN vcpkg.exe update +RUN vcpkg.exe install \ + boost-interprocess:x64-windows \ + boost-stacktrace:x64-windows \ + b64:x64-windows \ + openssl-windows:x64-windows \ + openssl:x64-windows \ + pthread:x64-windows \ + rapidjson:x64-windows \ + zlib:x64-windows +RUN vcpkg.exe integrate install + +LABEL VCPGK_VERSION=${VCPGK_VERSION} + +WORKDIR / + +# +# Installing CUDA +# +ARG CUDA_MAJOR=12 +ARG CUDA_MINOR=8 +ARG CUDA_PATCH=0 +ARG CUDA_VERSION=${CUDA_MAJOR}.${CUDA_MINOR}.${CUDA_PATCH} +ARG CUDA_PACKAGES="nvcc_${CUDA_MAJOR}.${CUDA_MINOR} \ + cudart_${CUDA_MAJOR}.${CUDA_MINOR} \ + nvml_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + nvrtc_${CUDA_MAJOR}.${CUDA_MINOR} nvrtc_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + cublas_${CUDA_MAJOR}.${CUDA_MINOR} cublas_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + cufft_${CUDA_MAJOR}.${CUDA_MINOR} cufft_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + curand_${CUDA_MAJOR}.${CUDA_MINOR} curand_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + cusolver_${CUDA_MAJOR}.${CUDA_MINOR} cusolver_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + cusparse_${CUDA_MAJOR}.${CUDA_MINOR} cusparse_dev_${CUDA_MAJOR}.${CUDA_MINOR} \ + cupti_${CUDA_MAJOR}.${CUDA_MINOR} \ + thrust_${CUDA_MAJOR}.${CUDA_MINOR} \ + visual_studio_integration_${CUDA_MAJOR}.${CUDA_MINOR}" +ARG CUDA_INSTALL_ROOT_WP="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_MAJOR}.${CUDA_MINOR}" + +ARG CUDA_SOURCE=https://developer.download.nvidia.com/compute/cuda/${CUDA_VERSION}/network_installers/cuda_${CUDA_VERSION}_windows_network.exe +ADD ${CUDA_SOURCE} cuda_${CUDA_VERSION}_windows_network.exe + +RUN cuda_%CUDA_VERSION%_windows_network.exe -s %CUDA_PACKAGES% +# Copy the CUDA visualstudio integration from where it was installed +# into the appropriate place in BuildTools +RUN copy "%CUDA_INSTALL_ROOT_WP%\extras\visual_studio_integration\MSBuildExtensions\*" "%VS_INSTALL_PATH_WP%\MSBuild\Microsoft\VC\v170\BuildCustomizations" + +RUN setx PATH "%CUDA_INSTALL_ROOT_WP%\bin;%PATH%" + +ENV CUDA_VERSION=${CUDA_VERSION} +LABEL CUDA_VERSION="${CUDA_VERSION}" + +ARG CUDNN_VERSION=9.7.1.26 +ENV CUDNN_VERSION ${CUDNN_VERSION} +COPY --from=dependency_base /cudnn /cudnn +RUN copy cudnn\bin\cudnn*.dll "%CUDA_INSTALL_ROOT_WP%\bin\." +RUN copy cudnn\lib\x64\cudnn*.lib "%CUDA_INSTALL_ROOT_WP%\lib\x64\." +RUN copy cudnn\include\cudnn*.h "%CUDA_INSTALL_ROOT_WP%\include\." +LABEL CUDNN_VERSION="${CUDNN_VERSION}" + +ARG TENSORRT_VERSION=10.8.0.43 +ENV TRT_VERSION ${TENSORRT_VERSION} +COPY --from=dependency_base /TensorRT /TensorRT +RUN setx PATH "c:\TensorRT\lib;%PATH%" +LABEL TENSORRT_VERSION="${TENSORRT_VERSION}" + +# It is important that the entrypoint initialize VisualStudio +# environment otherwise the build will fail. Also set +# CMAKE_TOOLCHAIN_FILE and VCPKG_TARGET_TRIPLET so +# that cmake can find the packages installed by vcpkg. +ENTRYPOINT C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat && diff --git a/LICENSE b/LICENSE index 6c65d3b34d..d367cc74ad 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved. +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/README.md b/README.md index fdb6b2a5bf..65d4918be5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![License](https://img.shields.io/badge/License-BSD3-lightgrey.svg)](https://opensource.org/licenses/BSD-3-Clause) - ->[!WARNING] ->You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.69.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 26.05 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server Triton Inference Server is an open source inference serving software that streamlines AI inferencing. Triton enables teams to deploy any AI model from multiple deep learning and machine learning frameworks, including TensorRT, -PyTorch, ONNX, OpenVINO, Python, RAPIDS FIL, and more. Triton +TensorFlow, PyTorch, ONNX, OpenVINO, Python, RAPIDS FIL, and more. Triton Inference Server supports inference across cloud, data center, edge and embedded devices on NVIDIA GPUs, x86 and ARM CPU, or AWS Inferentia. Triton Inference Server delivers optimized performance for many query types, including real time, @@ -54,14 +48,14 @@ Major features include: frameworks](https://github.com/triton-inference-server/fil_backend) - [Concurrent model execution](docs/user_guide/architecture.md#concurrent-model-execution) -- [Dynamic batching](docs/user_guide/batcher.md#dynamic-batcher) -- [Sequence batching](docs/user_guide/batcher.md#sequence-batcher) and +- [Dynamic batching](docs/user_guide/model_configuration.md#dynamic-batcher) +- [Sequence batching](docs/user_guide/model_configuration.md#sequence-batcher) and [implicit state management](docs/user_guide/architecture.md#implicit-state-management) for stateful models - Provides [Backend API](https://github.com/triton-inference-server/backend) that allows adding custom backends and pre/post processing operations - Supports writing custom backends in python, a.k.a. - [Python-based backends.](https://github.com/triton-inference-server/backend/blob/main/docs/python_based_backends.md#python-based-backends) + [Python-based backends.](https://github.com/triton-inference-server/backend/blob/r25.03/docs/python_based_backends.md#python-based-backends) - Model pipelines using [Ensembling](docs/user_guide/architecture.md#ensemble-models) or [Business Logic Scripting @@ -70,8 +64,8 @@ Major features include: protocols](docs/customization_guide/inference_protocols.md) based on the community developed [KServe protocol](https://github.com/kserve/kserve/tree/master/docs/predict-api/v2) -- A [C API](docs/customization_guide/inprocess_c_api.md) and - [Java API](docs/customization_guide/inprocess_java_api.md) +- A [C API](docs/customization_guide/inference_protocols.md#in-process-triton-server-api) and + [Java API](docs/customization_guide/inference_protocols.md#java-bindings-for-in-process-triton-server-api) allow Triton to link directly into your application for edge and other in-process use cases - [Metrics](docs/user_guide/metrics.md) indicating GPU utilization, server throughput, server latency, and more @@ -90,16 +84,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r26.05 https://github.com/triton-inference-server/server.git +git clone -b r25.03 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:26.05-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.03-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:26.05-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.03-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': @@ -134,6 +128,7 @@ images. - [Install Triton Inference Server without Docker containers](docs/customization_guide/build.md#building-without-docker) - [Build a custom Triton Inference Server Docker container](docs/customization_guide/compose.md) - [Build Triton Inference Server from source](docs/customization_guide/build.md#building-on-unsupported-platforms) +- [Build Triton Inference Server for Windows 10](docs/customization_guide/build.md#building-for-windows-10) - Examples for deploying Triton Inference Server with Kubernetes and Helm on [GCP](deploy/gcp/README.md), [AWS](deploy/aws/README.md), and [NVIDIA FleetCommand](deploy/fleetcommand/README.md) - [Secure Deployment Considerations](docs/customization_guide/deploy.md) @@ -165,16 +160,17 @@ configuration](docs/user_guide/model_configuration.md) for the model. - Triton supports multiple execution engines, called [backends](https://github.com/triton-inference-server/backend#where-can-i-find-all-the-backends-that-are-available-for-triton), including [TensorRT](https://github.com/triton-inference-server/tensorrt_backend), + [TensorFlow](https://github.com/triton-inference-server/tensorflow_backend), [PyTorch](https://github.com/triton-inference-server/pytorch_backend), [ONNX](https://github.com/triton-inference-server/onnxruntime_backend), [OpenVINO](https://github.com/triton-inference-server/openvino_backend), [Python](https://github.com/triton-inference-server/python_backend), and more - Not all the above backends are supported on every platform supported by Triton. Look at the - [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/main/docs/backend_platform_support_matrix.md) + [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/r25.03/docs/backend_platform_support_matrix.md) to learn which backends are supported on your target platform. - Learn how to [optimize performance](docs/user_guide/optimization.md) using the - [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/main/README.md) + [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/r25.03/README.md) and [Model Analyzer](https://github.com/triton-inference-server/model_analyzer) - Learn how to [manage loading and unloading models](docs/user_guide/model_management.md) in @@ -188,14 +184,14 @@ A Triton *client* application sends inference and other requests to Triton. The [Python and C++ client libraries](https://github.com/triton-inference-server/client) provide APIs to simplify this communication. -- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/examples), - [Python](https://github.com/triton-inference-server/client/blob/main/src/python/examples), - and [Java](https://github.com/triton-inference-server/client/blob/main/src/java/src/main/java/triton/client/examples) +- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/r25.03/src/c%2B%2B/examples), + [Python](https://github.com/triton-inference-server/client/blob/r25.03/src/python/examples), + and [Java](https://github.com/triton-inference-server/client/blob/r25.03/src/java/src/main/java/triton/client/examples) - Configure [HTTP](https://github.com/triton-inference-server/client#http-options) and [gRPC](https://github.com/triton-inference-server/client#grpc-options) client options - Send input data (e.g. a jpeg image) directly to Triton in the [body of an HTTP - request without any additional metadata](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_binary_data.md#raw-binary-request) + request without any additional metadata](https://github.com/triton-inference-server/server/blob/r25.03/docs/protocol/extension_binary_data.md#raw-binary-request) ### Extend Triton @@ -204,7 +200,7 @@ designed for modularity and flexibility - [Customize Triton Inference Server container](docs/customization_guide/compose.md) for your use case - [Create custom backends](https://github.com/triton-inference-server/backend) - in either [C/C++](https://github.com/triton-inference-server/backend/blob/main/README.md#triton-backend-api) + in either [C/C++](https://github.com/triton-inference-server/backend/blob/r25.03/README.md#triton-backend-api) or [Python](https://github.com/triton-inference-server/python_backend) - Create [decoupled backends and models](docs/user_guide/decoupled_models.md) that can send multiple responses for a request or not send any responses for a request @@ -258,3 +254,4 @@ For questions, we recommend posting in our community Please refer to the [NVIDIA Developer Triton page](https://developer.nvidia.com/nvidia-triton-inference-server) for more information. + diff --git a/TRITON_VERSION b/TRITON_VERSION index 6a166a54c5..5f46e11eed 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.70.0dev +2.56.0 diff --git a/build.py b/build.py index 9111c06c3a..6ea96f5218 100755 --- a/build.py +++ b/build.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -71,13 +71,14 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.70.0dev", - "triton_container_version": "26.06dev", - "upstream_container_version": "26.05", - "ort_version": "1.24.4", - "ort_openvino_version": "2026.1.0", - "standalone_openvino_version": "2026.1.0", - "dcgm_version": "4.5.3-1", + "release_version": "2.56.0", + "triton_container_version": "25.03", + "upstream_container_version": "25.03", + "ort_version": "1.21.0", + "ort_openvino_version": "2025.0.0", + "standalone_openvino_version": "2025.0.0", + "dcgm_version": "3.3.6", + "vllm_version": "0.7.3", "rhel_py_version": "3.12.3", } @@ -118,8 +119,6 @@ def fail_if(p, msg): def target_platform(): # When called by compose.py, FLAGS will be None if FLAGS and FLAGS.target_platform is not None: - if FLAGS.target_platform == "windows": - fail("Windows is no longer a supported target platform.") return FLAGS.target_platform platform_string = platform.system().lower() if platform_string == "linux": @@ -130,8 +129,6 @@ def target_platform(): return "linux" else: return "rhel" - elif platform_string == "windows": - fail("Windows is no longer a supported target platform.") else: return platform_string @@ -171,6 +168,13 @@ def __del__(self): def close(self): if self._file is not None: + if target_platform() == "windows": + self.blankln() + self._file.write("}\n") + self._file.write("catch {\n") + self._file.write(" $_;\n") + self._file.write(" ExitWithCode 1;\n") + self._file.write("}\n") """Close the file""" self._file.close() self._file = None @@ -198,7 +202,8 @@ def comment_verbose(self, msg=""): self.comment(msg) def header(self, desc=None): - self._file.write("#!/usr/bin/env bash\n\n") + if target_platform() != "windows": + self._file.write("#!/usr/bin/env bash\n\n") if desc is not None: self.comment() @@ -207,12 +212,27 @@ def header(self, desc=None): self.blankln() self.comment("Exit script immediately if any command fails") - self._file.write("set -e\n") - if self._verbose: - self._file.write("set -x\n") + if target_platform() == "windows": + self._file.write("$UseStructuredOutput = $false\n") + self.blankln() + self._file.write("function ExitWithCode($exitcode) {\n") + self._file.write(" $host.SetShouldExit($exitcode)\n") + self._file.write(" exit $exitcode\n") + self._file.write("}\n") + self.blankln() + if self._verbose: + self._file.write("Set-PSDebug -Trace 1\n") + self.blankln() + self._file.write("try {\n") + else: + self._file.write("set -e\n") + if self._verbose: + self._file.write("set -x\n") self.blankln() def envvar_ref(self, v): + if target_platform() == "windows": + return f"${{env:{v}}}" return f"${{{v}}}" def cmd(self, clist, check_exitcode=False): @@ -223,23 +243,54 @@ def cmd(self, clist, check_exitcode=False): self._file.write(f"{c} ") self.blankln() + if check_exitcode: + if target_platform() == "windows": + self._file.write("if ($LASTEXITCODE -ne 0) {\n") + self._file.write( + ' Write-Output "exited with status code $LASTEXITCODE";\n' + ) + self._file.write(" ExitWithCode 1;\n") + self._file.write("}\n") + def cwd(self, path): - self.cmd(f"cd {path}") + if target_platform() == "windows": + self.cmd(f"Set-Location -EV Err -EA Stop {path}") + else: + self.cmd(f"cd {path}") def cp(self, src, dest): - self.cmd(f"cp {src} {dest}") + if target_platform() == "windows": + self.cmd(f"Copy-Item -EV Err -EA Stop {src} -Destination {dest}") + else: + self.cmd(f"cp {src} {dest}") def mkdir(self, path): - self.cmd(f"mkdir -p {pathlib.Path(path)}") + if target_platform() == "windows": + self.cmd( + f"New-Item -EV Err -EA Stop -ItemType Directory -Force -Path {path}" + ) + else: + self.cmd(f"mkdir -p {pathlib.Path(path)}") def rmdir(self, path): - self.cmd(f"rm -fr {pathlib.Path(path)}") + if target_platform() == "windows": + self.cmd(f"if (Test-Path -Path {path}) {{") + self.cmd(f" Remove-Item -EV Err -EA Stop -Recurse -Force {path}") + self.cmd("}") + else: + self.cmd(f"rm -fr {pathlib.Path(path)}") def cpdir(self, src, dest): - self.cmd(f"cp -r {src} {dest}") + if target_platform() == "windows": + self.cmd(f"Copy-Item -EV Err -EA Stop -Recurse {src} -Destination {dest}") + else: + self.cmd(f"cp -r {src} {dest}") def tar(self, subdir, tar_filename): - self.cmd(f"tar zcf {tar_filename} {subdir}") + if target_platform() == "windows": + fail("unsupported operation: tar") + else: + self.cmd(f"tar zcf {tar_filename} {subdir}") def cmake(self, args): # Pass some additional envvars into cmake... @@ -259,7 +310,10 @@ def gitclone(self, repo, tag, subdir, org): if not FLAGS.no_force_clone: self.rmdir(clone_dir) - self.cmd(f"if [[ ! -e {clone_dir} ]]; then") + if target_platform() == "windows": + self.cmd(f"if (-Not (Test-Path -Path {clone_dir})) {{") + else: + self.cmd(f"if [[ ! -e {clone_dir} ]]; then") # FIXME [DLIS-4045 - Currently the tag starting with "pull/" is not # working with "--repo-tag" as the option is not forwarded to the @@ -269,19 +323,19 @@ def gitclone(self, repo, tag, subdir, org): # reference onto a new branch we name "tritonbuildref". if tag.startswith("pull/"): self.cmd( - f" git clone --recursive --depth=1 {org}/{repo}.git {subdir}; git --git-dir {subdir}/.git log --oneline -1", + f" git clone --recursive --depth=1 {org}/{repo}.git {subdir};", check_exitcode=True, ) - self.cmd("fi") + self.cmd("}" if target_platform() == "windows" else "fi") self.cwd(subdir) self.cmd(f"git fetch origin {tag}:tritonbuildref", check_exitcode=True) self.cmd(f"git checkout tritonbuildref", check_exitcode=True) else: self.cmd( - f" git clone --recursive --single-branch --depth=1 -b {tag} {org}/{repo}.git {subdir}; git --git-dir {subdir}/.git log --oneline -1", + f" git clone --recursive --single-branch --depth=1 -b {tag} {org}/{repo}.git {subdir};", check_exitcode=True, ) - self.cmd("fi") + self.cmd("}" if target_platform() == "windows" else "fi") def cmake_core_arg(name, type, value): @@ -444,6 +498,7 @@ def core_cmake_args(components, backends, cmake_dir, install_dir): cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends)) cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends)) + cargs.append(cmake_core_enable("TRITON_ENABLE_MYSQL_ODBC", FLAGS.enable_mysql_odbc)) cargs += cmake_core_extra_args() cargs.append(cmake_dir) @@ -551,7 +606,14 @@ def backend_cmake_args(images, components, be, install_dir, library_paths): cmake_backend_enable(be, "TRITON_ENABLE_METRICS", FLAGS.enable_metrics) ) - if target_platform() == "igpu": + # [DLIS-4950] always enable below once Windows image is updated with CUPTI + # cargs.append(cmake_backend_enable(be, 'TRITON_ENABLE_MEMORY_TRACKER', True)) + if (target_platform() == "windows") and (not FLAGS.no_container_build): + print( + "Warning: Detected docker build is used for Windows, backend utility 'device memory tracker' will be disabled due to missing library in CUDA Windows docker image." + ) + cargs.append(cmake_backend_enable(be, "TRITON_ENABLE_MEMORY_TRACKER", False)) + elif target_platform() == "igpu": print( "Warning: Detected iGPU build, backend utility 'device memory tracker' will be disabled as iGPU doesn't contain required version of the library." ) @@ -561,7 +623,7 @@ def backend_cmake_args(images, components, be, install_dir, library_paths): cargs += cmake_backend_extra_args(be) if be == "tensorrtllm": - cargs.append("-S ../triton_backend/inflight_batcher_llm -B .") + cargs.append("-S ../inflight_batcher_llm -B .") else: cargs.append("..") @@ -599,10 +661,6 @@ def pytorch_cmake_args(images): cargs.append( cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx) ) - if target_platform() == "igpu": - cargs.append( - cmake_backend_enable("pytorch", "TRITON_PYTORCH_NVSHMEM", False) - ) return cargs @@ -630,51 +688,59 @@ def onnxruntime_cmake_args(images, library_paths): ) ) - if "base" in images: - cargs.append( - cmake_backend_arg( - "onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"] + if target_platform() == "windows": + if "base" in images: + cargs.append( + cmake_backend_arg( + "onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"] + ) ) - ) else: - cargs.append( - cmake_backend_arg( - "onnxruntime", - "TRITON_BUILD_CONTAINER_VERSION", - None, - FLAGS.upstream_container_version, + if "base" in images: + cargs.append( + cmake_backend_arg( + "onnxruntime", "TRITON_BUILD_CONTAINER", None, images["base"] + ) + ) + else: + cargs.append( + cmake_backend_arg( + "onnxruntime", + "TRITON_BUILD_CONTAINER_VERSION", + None, + FLAGS.triton_container_version, + ) ) - ) - # TODO: TPRD-333 OpenVino extension is not currently supported by our manylinux build - if ( - (target_machine() != "aarch64") - and (target_platform() != "rhel") - and (FLAGS.ort_openvino_version is not None) - ): - cargs.append( - cmake_backend_enable( - "onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", True + # TODO: TPRD-333 OpenVino extension is not currently supported by our manylinux build + if ( + (target_machine() != "aarch64") + and (target_platform() != "rhel") + and (FLAGS.ort_openvino_version is not None) + ): + cargs.append( + cmake_backend_enable( + "onnxruntime", "TRITON_ENABLE_ONNXRUNTIME_OPENVINO", True + ) ) - ) - cargs.append( - cmake_backend_arg( - "onnxruntime", - "TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION", - None, - FLAGS.ort_openvino_version, + cargs.append( + cmake_backend_arg( + "onnxruntime", + "TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION", + None, + FLAGS.ort_openvino_version, + ) ) - ) - if (target_platform() == "igpu") or (target_platform() == "rhel"): - cargs.append( - cmake_backend_arg( - "onnxruntime", - "TRITON_BUILD_TARGET_PLATFORM", - None, - target_platform(), + if (target_platform() == "igpu") or (target_platform() == "rhel"): + cargs.append( + cmake_backend_arg( + "onnxruntime", + "TRITON_BUILD_TARGET_PLATFORM", + None, + target_platform(), + ) ) - ) return cargs @@ -688,21 +754,29 @@ def openvino_cmake_args(): FLAGS.standalone_openvino_version, ) ] - if "base" in images: - cargs.append( - cmake_backend_arg( - "openvino", "TRITON_BUILD_CONTAINER", None, images["base"] + if target_platform() == "windows": + if "base" in images: + cargs.append( + cmake_backend_arg( + "openvino", "TRITON_BUILD_CONTAINER", None, images["base"] + ) ) - ) else: - cargs.append( - cmake_backend_arg( - "openvino", - "TRITON_BUILD_CONTAINER_VERSION", - None, - FLAGS.upstream_container_version, + if "base" in images: + cargs.append( + cmake_backend_arg( + "openvino", "TRITON_BUILD_CONTAINER", None, images["base"] + ) + ) + else: + cargs.append( + cmake_backend_arg( + "openvino", + "TRITON_BUILD_CONTAINER_VERSION", + None, + FLAGS.upstream_container_version, + ) ) - ) return cargs @@ -710,6 +784,12 @@ def tensorrt_cmake_args(): cargs = [ cmake_backend_enable("tensorrt", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx), ] + if target_platform() == "windows": + cargs.append( + cmake_backend_arg( + "tensorrt", "TRITON_TENSORRT_INCLUDE_PATHS", None, "c:/TensorRT/include" + ) + ) return cargs @@ -779,12 +859,9 @@ def install_dcgm_libraries(dcgm_version, target_machine): # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo \\ && dnf clean expire-cache \\ - && dnf makecache --refresh \\ - && dnf install --assumeyes \\ - datacenter-gpu-manager-4-core-1:{} \\ - datacenter-gpu-manager-4-devel-1:{} + && dnf install -y datacenter-gpu-manager-{} """.format( - dcgm_version, dcgm_version, dcgm_version + dcgm_version, dcgm_version ) else: return """ @@ -792,12 +869,9 @@ def install_dcgm_libraries(dcgm_version, target_machine): # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo \\ && dnf clean expire-cache \\ - && dnf makecache --refresh \\ - && dnf install --assumeyes \\ - datacenter-gpu-manager-4-core-1:{} \\ - datacenter-gpu-manager-4-devel-1:{} + && dnf install -y datacenter-gpu-manager-{} """.format( - dcgm_version, dcgm_version, dcgm_version + dcgm_version, dcgm_version ) else: if target_machine == "aarch64": @@ -808,12 +882,10 @@ def install_dcgm_libraries(dcgm_version, target_machine): https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb \\ && apt install /tmp/cuda-keyring.deb \\ && rm /tmp/cuda-keyring.deb \\ - && apt update -qq \\ - && apt install --yes --no-install-recommends \\ - datacenter-gpu-manager-4-core=1:{} \\ - datacenter-gpu-manager-4-dev=1:{} + && apt-get update \\ + && apt-get install -y datacenter-gpu-manager=1:{} """.format( - dcgm_version, dcgm_version, dcgm_version + dcgm_version, dcgm_version ) else: return """ @@ -823,16 +895,20 @@ def install_dcgm_libraries(dcgm_version, target_machine): https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \\ && apt install /tmp/cuda-keyring.deb \\ && rm /tmp/cuda-keyring.deb \\ - && apt update -qq \\ - && apt install --yes --no-install-recommends \\ - datacenter-gpu-manager-4-core=1:{} \\ - datacenter-gpu-manager-4-dev=1:{} + && apt-get update \\ + && apt-get install -y datacenter-gpu-manager=1:{} """.format( - dcgm_version, dcgm_version, dcgm_version + dcgm_version, dcgm_version ) def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): + buildbase_odbc_layer = "" + if FLAGS.enable_mysql_odbc: + buildbase_odbc_layer = """ +RUN yum install -y unixODBC-devel +""" + df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} @@ -848,7 +924,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION -ENV PIP_BREAK_SYSTEM_PACKAGES=1 CMAKE_POLICY_VERSION_MINIMUM=3.5 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 """ df += """ # Install docker docker buildx @@ -890,13 +966,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): xz-devel \\ zlib-devel """ - if argmap["NVIDIA_BUILD_ID"] is not None: - df += """ -ENV BUILD_NUMBER={} -""".format( - argmap["NVIDIA_BUILD_ID"] - ) - + df += buildbase_odbc_layer if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"): df += """ RUN curl -k -s -L https://github.com/ccache/ccache/archive/refs/tags/v4.10.2.tar.gz -o /tmp/ccache.tar.gz \\ @@ -926,18 +996,25 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): setuptools \\ docker \\ virtualenv \\ - patchelf==0.17.2 \\ - cmake==4.0.3 -""" - df += f""" + patchelf==0.17.2 + # Install boost version >= 1.78 for boost::span # Current libboost-dev apt packages are < 1.78, so install from tar.gz -RUN wget -O /tmp/boost.tar.gz {FLAGS.boost_url} \\ - && sha256sum /tmp/boost.tar.gz | grep {FLAGS.boost_sha256} \\ +RUN wget -O /tmp/boost.tar.gz \\ + https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz \\ && (cd /tmp && tar xzf boost.tar.gz) \\ && mv /tmp/boost_1_80_0/boost /usr/include/boost -""" +# Server build requires recent version of CMake (FetchContent required) +# Might not need this if the installed version of cmake is high enough for our build. +# RUN apt update -q=2 \\ +# && apt install -y gpg wget \\ +# && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\ +# && . /etc/os-release \\ +# && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\ +# && apt-get update -q=2 \\ +# && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* +""" if FLAGS.enable_gpu: df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine()) df += """ @@ -972,15 +1049,20 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): ARG TRITON_VERSION ARG TRITON_CONTAINER_VERSION -ENV PIP_BREAK_SYSTEM_PACKAGES=1 CMAKE_POLICY_VERSION_MINIMUM=3.5 +ENV PIP_BREAK_SYSTEM_PACKAGES=1 """ - if argmap["NVIDIA_BUILD_ID"] is not None: + # Install the windows- or linux-specific buildbase dependencies + if target_platform() == "windows": df += """ -ENV BUILD_NUMBER={} -""".format( - argmap["NVIDIA_BUILD_ID"] +RUN python3 -m pip install build + +SHELL ["cmd", "/S", "/C"] +""" + else: + mysql_odbc_line = ( + " unixodbc-dev \\\n" if FLAGS.enable_mysql_odbc else "" ) - df += """ + df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive @@ -1030,29 +1112,36 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): libarchive-dev \\ libxml2-dev \\ libnuma-dev \\ - wget \\ +""" + df += mysql_odbc_line + df += """ wget \\ && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ build \\ docker \\ virtualenv \\ - patchelf==0.17.2 \\ - cmake==4.0.3 \\ - pybind11[global] -""" + patchelf==0.17.2 - df += f""" # Install boost version >= 1.78 for boost::span # Current libboost-dev apt packages are < 1.78, so install from tar.gz -RUN wget -O /tmp/boost.tar.gz {FLAGS.boost_url} \\ - && sha256sum /tmp/boost.tar.gz | grep {FLAGS.boost_sha256} \\ +RUN wget -O /tmp/boost.tar.gz \\ + https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz \\ && (cd /tmp && tar xzf boost.tar.gz) \\ && mv /tmp/boost_1_80_0/boost /usr/include/boost + +# Server build requires recent version of CMake (FetchContent required) +RUN apt update -q=2 \\ + && apt install -y gpg wget \\ + && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \\ + && . /etc/os-release \\ + && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\ + && apt-get update -q=2 \\ + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* """ - if FLAGS.enable_gpu: - df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine()) + if FLAGS.enable_gpu: + df += install_dcgm_libraries(argmap["DCGM_VERSION"], target_machine()) df += """ ENV TRITON_SERVER_VERSION ${TRITON_VERSION} @@ -1076,7 +1165,14 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): # Copy in the triton source. We remove existing contents first in # case the FROM container has something there already. - df += """ + if target_platform() == "windows": + df += """ +WORKDIR /workspace +RUN rmdir /S/Q * || exit 0 +COPY . . +""" + else: + df += """ WORKDIR /workspace RUN rm -fr * COPY . . @@ -1123,19 +1219,12 @@ def create_dockerfile_linux( df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} +ARG BASE_IMAGE={} + """.format( argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], - ) - if "vllm" in backends and argmap["INFERENCE_IMAGE"] is None: - argmap[ - "INFERENCE_IMAGE" - ] = f"nvcr.io/nvidia/vllm:{FLAGS.upstream_container_version}-py3" - df += """ARG BASE_IMAGE={} -""".format( - argmap["INFERENCE_IMAGE"] - if argmap["INFERENCE_IMAGE"] is not None - else argmap["BASE_IMAGE"], + argmap["BASE_IMAGE"], ) # PyTorch backends need extra CUDA and other @@ -1165,16 +1254,16 @@ def create_dockerfile_linux( argmap, backends, FLAGS.enable_gpu, target_machine() ) - df += f""" + df += """ WORKDIR /opt -COPY build/install tritonserver +COPY --chown=1000:1000 build/install tritonserver WORKDIR /opt/tritonserver -COPY NVIDIA_Deep_Learning_Container_License.pdf . +COPY --chown=1000:1000 NVIDIA_Deep_Learning_Container_License.pdf . RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonserver-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] && \\ + "tritonserver-*.whl" | xargs -I {} pip install --upgrade {}[all] && \\ find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonfrontend-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] + "tritonfrontend-*.whl" | xargs -I {} pip install --upgrade {}[all] RUN pip3 install -r python/openai/requirements.txt @@ -1185,20 +1274,36 @@ def create_dockerfile_linux( df += """ LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true LABEL com.amazonaws.sagemaker.capabilities.multi-models=true -COPY docker/sagemaker/serve /usr/bin/. +COPY --chown=1000:1000 docker/sagemaker/serve /usr/bin/. """ # This is required since libcublasLt.so is not present during the build # stage of the PyTorch backend if not FLAGS.enable_gpu and ("pytorch" in backends): df += """ -RUN patchelf --add-needed /usr/local/cuda/lib64/stubs/libcublasLt.so.13 backends/pytorch/libtorch_cuda.so +RUN patchelf --add-needed /usr/local/cuda/lib64/stubs/libcublasLt.so.12 backends/pytorch/libtorch_cuda.so """ if "tensorrtllm" in backends: df += """ +# Install required packages for TRT-LLM models +# Remove contents that are not needed in runtime +# Setuptools has breaking changes in version 70.0.0, so fix it to 69.5.1 +# The generated code in grpc_service_pb2_grpc.py depends on grpcio>=1.64.0, so fix it to 1.64.0 RUN ldconfig && \\ - find /opt/tritonserver -name lib*so -exec dirname {} \\; > /etc/ld.so.conf.d/tritonserver.conf && \\ - ldconfig - + ARCH="$(uname -i)" && \\ + rm -fr ${TRT_ROOT}/bin ${TRT_ROOT}/targets/${ARCH}-linux-gnu/bin ${TRT_ROOT}/data && \\ + rm -fr ${TRT_ROOT}/doc ${TRT_ROOT}/onnx_graphsurgeon ${TRT_ROOT}/python && \\ + rm -fr ${TRT_ROOT}/samples ${TRT_ROOT}/targets/${ARCH}-linux-gnu/samples && \\ + pip3 install --no-cache-dir transformers && \\ + find /usr -name libtensorrt_llm.so -exec dirname {} \; > /etc/ld.so.conf.d/tensorrt-llm.conf && \\ + find /opt/tritonserver -name libtritonserver.so -exec dirname {} \; > /etc/ld.so.conf.d/triton-tensorrtllm-worker.conf && \\ + pip3 install --no-cache-dir grpcio-tools==1.64.0 && \\ + pip3 uninstall -y setuptools +ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH + +# There are some ucc issues when spawning mpi processes with ompi v4.1.7a1. +# Downgrade to ompi v4.1.5rc2 to avoid the issue. +RUN rm -fr /opt/hpcx/ompi +COPY --from=nvcr.io/nvidia/tritonserver:24.02-py3-min /opt/hpcx/ompi /opt/hpcx/ompi """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) @@ -1248,9 +1353,8 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV TRITON_SERVER_GPU_ENABLED {gpu_enabled} # Create a user that can be used to run triton as -# non-root. Make sure that this user is given ID 1000. Server -# artifacts copied below remain owned by root; the triton-server -# user reads and executes them via standard group/other permissions. +# non-root. Make sure that this user to given ID 1000. All server +# artifacts copied below are assign to this user. ENV TRITON_SERVER_USER=triton-server RUN userdel tensorrt-server > /dev/null 2>&1 || true \\ && userdel ubuntu > /dev/null 2>&1 || true \\ @@ -1276,7 +1380,7 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach libb64-devel \\ gperftools-devel \\ wget \\ - python3.12-pip \\ + python3-pip \\ numactl-devel RUN pip3 install patchelf==0.17.2 @@ -1301,6 +1405,7 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach libgoogle-perftools-dev \\ libjemalloc-dev \\ libnuma-dev \\ + software-properties-common \\ wget \\ {backend_dependencies} \\ python3-pip \\ @@ -1378,14 +1483,40 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach virtualenv \\ && rm -rf /var/lib/apt/lists/* """ - if "tensorrtllm" in backends or "vllm" in backends: + if "tensorrtllm" in backends: df += """ -ENV TRITON_CUDACRT_PATH=/usr/local/cuda/include \\ - TRITON_CUDART_PATH=/usr/local/cuda/include \\ - TRITON_CUOBJDUMP_PATH=/usr/local/cuda/bin/cuobjdump \\ - TRITON_CUPTI_PATH=/usr/local/cuda/include \\ - TRITON_NVDISASM_PATH=/usr/local/cuda/bin/nvdisasm \\ - TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas +# Updating the openssh-client to fix for the CVE-2024-6387. This can be removed when trtllm uses a later CUDA container(12.5 or later) +RUN apt-get update \\ + && apt-get install -y --no-install-recommends \\ + openssh-client \\ + && rm -rf /var/lib/apt/lists/* + """ + + if "vllm" in backends: + df += f""" +ARG BUILD_PUBLIC_VLLM="true" +ARG VLLM_INDEX_URL +ARG PYTORCH_TRITON_URL + +RUN --mount=type=secret,id=req,target=/run/secrets/requirements \\ + if [ "$BUILD_PUBLIC_VLLM" = "false" ]; then \\ + pip3 install --no-cache-dir \\ + mkl==2021.1.1 \\ + mkl-include==2021.1.1 \\ + mkl-devel==2021.1.1 \\ + && pip3 install --no-cache-dir --progress-bar on --index-url $VLLM_INDEX_URL -r /run/secrets/requirements \\ + # Need to install in-house build of pytorch-triton to support triton_key definition used by torch 2.5.1 + && cd /tmp \\ + && wget $PYTORCH_TRITON_URL \\ + && pip install --no-cache-dir /tmp/pytorch_triton-*.whl \\ + && rm /tmp/pytorch_triton-*.whl; \\ + else \\ + # public vLLM needed for vLLM backend + pip3 install vllm=={DEFAULT_TRITON_VERSION_MAP["vllm_version"]}; \\ + fi + +ARG PYVER=3.12 +ENV LD_LIBRARY_PATH /usr/local/lib:/usr/local/lib/python${{PYVER}}/dist-packages/torch/lib:${{LD_LIBRARY_PATH}} """ if "dali" in backends: @@ -1394,18 +1525,6 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ - if target_platform() == "rhel": - repo_arch = "sbsa" if target_machine == "aarch64" else "x86_64" - df += """ -RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/{repo_arch}/cuda-rhel8.repo \\ - && dnf clean expire-cache \\ - && dnf install --assumeyes libnvshmem3-cuda-13 - -RUN dirname $(find /usr -name "libcudart*.so" -o -name "libnvinf*.so" -o -name "libnvshm*" -type f) | sort -u > /etc/ld.so.conf.d/triton-cuda-libs.conf && ldconfig -""".format( - repo_arch=repo_arch - ) - df += """ WORKDIR /opt/tritonserver RUN rm -fr /opt/tritonserver/* @@ -1445,20 +1564,18 @@ def add_cpu_libs_to_linux_dockerfile(backends, target_machine): df += """ RUN mkdir -p /usr/local/cuda/lib64/stubs COPY --from=min_container /usr/local/cuda/lib64/stubs/libcusparse.so /usr/local/cuda/lib64/stubs/libcusparse.so.12 -COPY --from=min_container /usr/local/cuda/lib64/stubs/libcusolver.so /usr/local/cuda/lib64/stubs/libcusolver.so.12 +COPY --from=min_container /usr/local/cuda/lib64/stubs/libcusolver.so /usr/local/cuda/lib64/stubs/libcusolver.so.11 COPY --from=min_container /usr/local/cuda/lib64/stubs/libcurand.so /usr/local/cuda/lib64/stubs/libcurand.so.10 -COPY --from=min_container /usr/local/cuda/lib64/stubs/libcufft.so /usr/local/cuda/lib64/stubs/libcufft.so.12 -COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublas.so /usr/local/cuda/lib64/stubs/libcublas.so.13 -COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublasLt.so /usr/local/cuda/lib64/stubs/libcublasLt.so.13 +COPY --from=min_container /usr/local/cuda/lib64/stubs/libcufft.so /usr/local/cuda/lib64/stubs/libcufft.so.11 +COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublas.so /usr/local/cuda/lib64/stubs/libcublas.so.12 +COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublasLt.so /usr/local/cuda/lib64/stubs/libcublasLt.so.12 +COPY --from=min_container /usr/local/cuda/lib64/stubs/libcublasLt.so /usr/local/cuda/lib64/stubs/libcublasLt.so.11 RUN mkdir -p /usr/local/cuda/targets/{cuda_arch}-linux/lib -COPY --from=min_container /usr/local/cuda/lib64/libcudart.so.13 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libcupti.so.13 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libnvJitLink.so.13 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libcufile.so.0 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libnvrtc.so.13 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libcusparseLt.so.0 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. -COPY --from=min_container /usr/local/cuda/lib64/libnvshmem_host.so.3 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. +COPY --from=min_container /usr/local/cuda/lib64/libcudart.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. +COPY --from=min_container /usr/local/cuda/lib64/libcupti.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. +COPY --from=min_container /usr/local/cuda/lib64/libnvToolsExt.so.1 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. +COPY --from=min_container /usr/local/cuda/lib64/libnvJitLink.so.12 /usr/local/cuda/targets/{cuda_arch}-linux/lib/. RUN mkdir -p /opt/hpcx/ucc/lib/ /opt/hpcx/ucx/lib/ COPY --from=min_container /opt/hpcx/ucc/lib/libucc.so.1 /opt/hpcx/ucc/lib/libucc.so.1 @@ -1518,6 +1635,55 @@ def change_default_python_version_rhel(version): return df +def create_dockerfile_windows( + ddir, dockerfile_name, argmap, backends, repoagents, caches +): + df = """ +ARG TRITON_VERSION={} +ARG TRITON_CONTAINER_VERSION={} +ARG BASE_IMAGE={} + +############################################################################ +## Production stage: Create container with just inference server executable +############################################################################ +FROM ${{BASE_IMAGE}} + +ARG TRITON_VERSION +ARG TRITON_CONTAINER_VERSION + +ENV TRITON_SERVER_VERSION ${{TRITON_VERSION}} +ENV NVIDIA_TRITON_SERVER_VERSION ${{TRITON_CONTAINER_VERSION}} +LABEL com.nvidia.tritonserver.version="${{TRITON_SERVER_VERSION}}" + +RUN setx path "%path%;C:\opt\tritonserver\bin" + +""".format( + argmap["TRITON_VERSION"], + argmap["TRITON_CONTAINER_VERSION"], + argmap["BASE_IMAGE"], + ) + df += """ +WORKDIR /opt +RUN rmdir /S/Q tritonserver || exit 0 +COPY --chown=1000:1000 build/install tritonserver + +WORKDIR /opt/tritonserver +COPY --chown=1000:1000 NVIDIA_Deep_Learning_Container_License.pdf . + +""" + df += """ +ENTRYPOINT [] +ENV NVIDIA_BUILD_ID {} +LABEL com.nvidia.build.id={} +LABEL com.nvidia.build.ref={} +""".format( + argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_ID"], argmap["NVIDIA_BUILD_REF"] + ) + + with open(os.path.join(ddir, dockerfile_name), "w") as dfile: + dfile.write(df) + + def create_build_dockerfiles( container_build_dir, images, backends, repoagents, caches, endpoints ): @@ -1527,6 +1693,8 @@ def create_build_dockerfiles( print( "warning: RHEL is not an officially supported target and you will probably experience errors attempting to build this container." ) + elif target_platform() == "windows": + base_image = "mcr.microsoft.com/dotnet/framework/sdk:4.8" elif target_platform() == "rhel": raise KeyError("A base image must be specified when targeting RHEL") elif FLAGS.enable_gpu: @@ -1536,24 +1704,22 @@ def create_build_dockerfiles( else: base_image = "ubuntu:24.04" - if "inference" in images: - inference_image = images["inference"] - else: - inference_image = None - dockerfileargmap = { "NVIDIA_BUILD_REF": "" if FLAGS.build_sha is None else FLAGS.build_sha, "NVIDIA_BUILD_ID": "" if FLAGS.build_id is None else FLAGS.build_id, "TRITON_VERSION": FLAGS.version, "TRITON_CONTAINER_VERSION": FLAGS.container_version, "BASE_IMAGE": base_image, - "INFERENCE_IMAGE": inference_image, "DCGM_VERSION": FLAGS.dcgm_version, } # For CPU-only image we need to copy some cuda libraries and dependencies # since we are using PyTorch containers that are not CPU-only. - if not FLAGS.enable_gpu and ("pytorch" in backends): + if ( + not FLAGS.enable_gpu + and ("pytorch" in backends) + and (target_platform() != "windows") + ): if "gpu-base" in images: gpu_base_image = images["gpu-base"] else: @@ -1571,15 +1737,25 @@ def create_build_dockerfiles( FLAGS.build_dir, "Dockerfile.buildbase", dockerfileargmap ) - create_dockerfile_linux( - FLAGS.build_dir, - "Dockerfile", - dockerfileargmap, - backends, - repoagents, - caches, - endpoints, - ) + if target_platform() == "windows": + create_dockerfile_windows( + FLAGS.build_dir, + "Dockerfile", + dockerfileargmap, + backends, + repoagents, + caches, + ) + else: + create_dockerfile_linux( + FLAGS.build_dir, + "Dockerfile", + dockerfileargmap, + backends, + repoagents, + caches, + endpoints, + ) # Dockerfile used for the creating the CI base image. create_dockerfile_cibase(FLAGS.build_dir, "Dockerfile.cibase", dockerfileargmap) @@ -1621,7 +1797,14 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ "--pull", ] - baseargs += ["--cache-from={}".format(k) for k in cachefrommap] + # Windows docker runs in a VM and memory needs to be specified + # explicitly (at least for some configurations of docker). + if target_platform() == "windows": + if FLAGS.container_memory: + baseargs += ["--memory", FLAGS.container_memory] + + if target_platform() != "windows": + baseargs += ["--cache-from={}".format(k) for k in cachefrommap] baseargs += ["."] @@ -1654,24 +1837,35 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ if not FLAGS.no_container_interactive: runargs += ["-it"] - runargs += ["-v", "/var/run/docker.sock:/var/run/docker.sock"] - if FLAGS.use_user_docker_config: - if os.path.exists(FLAGS.use_user_docker_config): - runargs += [ - "-v", - os.path.expanduser( - FLAGS.use_user_docker_config + ":/root/.docker/config.json" - ), - ] + if target_platform() == "windows": + if FLAGS.container_memory: + runargs += ["--memory", FLAGS.container_memory] + runargs += ["-v", "\\\\.\pipe\docker_engine:\\\\.\pipe\docker_engine"] + else: + runargs += ["-v", "/var/run/docker.sock:/var/run/docker.sock"] + if FLAGS.use_user_docker_config: + if os.path.exists(FLAGS.use_user_docker_config): + runargs += [ + "-v", + os.path.expanduser( + FLAGS.use_user_docker_config + ":/root/.docker/config.json" + ), + ] runargs += ["tritonserver_buildbase"] - runargs += ["./cmake_build"] + if target_platform() == "windows": + runargs += ["powershell.exe", "-noexit", "-File", "./cmake_build.ps1"] + else: + runargs += ["./cmake_build"] # Remove existing tritonserver_builder container... - docker_script._file.write( - 'if [ "$(docker ps -a | grep tritonserver_builder)" ]; then docker rm -f tritonserver_builder; fi\n' - ) + if target_platform() == "windows": + docker_script.cmd(["docker", "rm", "tritonserver_builder"]) + else: + docker_script._file.write( + 'if [ "$(docker ps -a | grep tritonserver_builder)" ]; then docker rm -f tritonserver_builder; fi\n' + ) docker_script.cmd(runargs, check_exitcode=True) @@ -1709,9 +1903,8 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ if secrets: finalargs += [ f"--secret id=req,src={requirements}", - f"--secret id=VLLM_INDEX_URL", - f"--secret id=PYTORCH_TRITON_URL", - f"--secret id=NVPL_SLIM_URL", + f"--build-arg VLLM_INDEX_URL={vllm_index_url}", + f"--build-arg PYTORCH_TRITON_URL={pytorch_triton_url}", f"--build-arg BUILD_PUBLIC_VLLM={build_public_vllm}", ] finalargs += [ @@ -1724,6 +1917,34 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(finalargs, check_exitcode=True) + if ( + FLAGS.pubmatic_bt_tritonserver_image + and target_platform() != "windows" + and target_platform() != "rhel" + ): + docker_script.blankln() + docker_script.commentln(8) + docker_script.comment( + "Pubmatic BT image: extend tritonserver with MySQL ODBC and config files" + ) + docker_script.comment( + "Uses Dockerfile.pubmatic-bt-tritonserver; requires replace-artifacts/" + ) + docker_script.comment() + pubmatic_args = [ + "docker", + "build", + "-t", + "pubmatic-bt-tritonserver", + "-f", + os.path.join( + THIS_SCRIPT_DIR, "Dockerfile.pubmatic-bt-tritonserver" + ), + ".", + ] + docker_script.cwd(THIS_SCRIPT_DIR) + docker_script.cmd(pubmatic_args, check_exitcode=True) + # # CI base image... tritonserver_cibase # @@ -1762,7 +1983,21 @@ def core_build( ) cmake_script.makeinstall() - if target_platform() == "rhel": + if target_platform() == "windows": + cmake_script.mkdir(os.path.join(install_dir, "bin")) + cmake_script.cp( + os.path.join(repo_install_dir, "bin", "tritonserver.exe"), + os.path.join(install_dir, "bin"), + ) + cmake_script.cp( + os.path.join(repo_install_dir, "bin", "tritonserver.dll"), + os.path.join(install_dir, "bin"), + ) + cmake_script.cp( + os.path.join(repo_install_dir, "lib", "tritonserver.lib"), + os.path.join(install_dir, "bin"), + ) + elif target_platform() == "rhel": cmake_script.mkdir(os.path.join(install_dir, "bin")) cmake_script.cp( os.path.join(repo_install_dir, "bin", "tritonserver"), @@ -1805,22 +2040,25 @@ def core_build( cmake_script.cp(os.path.join(repo_dir, "LICENSE"), install_dir) cmake_script.cp(os.path.join(repo_dir, "TRITON_VERSION"), install_dir) - # If requested, package the source code for all OSS used to build. - if ( - (not FLAGS.no_container_build) - and (not FLAGS.no_core_build) - and (not FLAGS.no_container_source) - ): - cmake_script.mkdir(os.path.join(install_dir, "third-party-src")) - cmake_script.cwd(repo_build_dir) - cmake_script.tar( - "third-party-src", - os.path.join(install_dir, "third-party-src", "src.tar.gz"), - ) - cmake_script.cp( - os.path.join(repo_dir, "docker", "README.third-party-src"), - os.path.join(install_dir, "third-party-src", "README"), - ) + # If requested, package the source code for all OSS used to build + # For windows, Triton is not delivered as a container so skip for + # windows platform. + if target_platform() != "windows": + if ( + (not FLAGS.no_container_build) + and (not FLAGS.no_core_build) + and (not FLAGS.no_container_source) + ): + cmake_script.mkdir(os.path.join(install_dir, "third-party-src")) + cmake_script.cwd(repo_build_dir) + cmake_script.tar( + "third-party-src", + os.path.join(install_dir, "third-party-src", "src.tar.gz"), + ) + cmake_script.cp( + os.path.join(repo_dir, "docker", "README.third-party-src"), + os.path.join(install_dir, "third-party-src", "README"), + ) cmake_script.comment() cmake_script.comment("end Triton core library and tritonserver executable") @@ -1873,11 +2111,7 @@ def backend_build( cmake_script.comment() cmake_script.mkdir(build_dir) cmake_script.cwd(build_dir) - if be == "tensorrtllm": - repository_name = "TensorRT-LLM" - cmake_script.gitclone(repository_name, tag, be, github_organization) - else: - cmake_script.gitclone(backend_repo(be), tag, be, github_organization) + cmake_script.gitclone(backend_repo(be), tag, be, github_organization) if be == "tensorrtllm": tensorrtllm_prebuild(cmake_script) @@ -2030,6 +2264,11 @@ def cibase_build( cmake_script.mkdir(ci_dir) + # On windows we are not yet using a CI/QA docker image for + # testing, so don't do anything... + if target_platform() == "windows": + return + # The core build produces some artifacts that are needed for CI # testing, so include those in the install. cmake_script.cpdir(os.path.join(repo_dir, "qa"), ci_dir) @@ -2058,9 +2297,12 @@ def cibase_build( cmake_script.mkdir(os.path.join(ci_dir, "backends")) for be in ("identity", "repeat", "square"): be_install_dir = os.path.join(build_dir, be, "install", "backends", be) - cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") + if target_platform() == "windows": + cmake_script.cmd(f"if (Test-Path -Path {be_install_dir}) {{") + else: + cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") cmake_script.cpdir(be_install_dir, os.path.join(ci_dir, "backends")) - cmake_script.cmd("fi") + cmake_script.cmd("}" if target_platform() == "windows" else "fi") # Some of the unit-test built backends are needed for CI testing cmake_script.mkdir(os.path.join(ci_dir, "tritonbuild", "tritonserver", "backends")) @@ -2073,12 +2315,15 @@ def cibase_build( "iterative_sequence", ): be_install_dir = os.path.join(repo_install_dir, "backends", be) - cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") + if target_platform() == "windows": + cmake_script.cmd(f"if (Test-Path -Path {be_install_dir}) {{") + else: + cmake_script.cmd(f"if [[ -e {be_install_dir} ]]; then") cmake_script.cpdir( be_install_dir, os.path.join(ci_dir, "tritonbuild", "tritonserver", "backends"), ) - cmake_script.cmd("fi") + cmake_script.cmd("}" if target_platform() == "windows" else "fi") # The onnxruntime_backend build produces some artifacts that # are needed for CI testing. @@ -2116,37 +2361,57 @@ def cibase_build( def finalize_build(cmake_script, install_dir, ci_dir): - cmake_script.cmd(f"chmod -R u+rwX,go+rX,go-w {install_dir}") - cmake_script.cmd(f"chmod -R u+rwX,go+rX,go-w {ci_dir}") + cmake_script.cmd(f"chmod -R a+rw {install_dir}") + cmake_script.cmd(f"chmod -R a+rw {ci_dir}") def enable_all(): - all_backends = [ - "ensemble", - "identity", - "square", - "repeat", - "onnxruntime", - "python", - "dali", - "pytorch", - "openvino", - "fil", - "tensorrt", - ] - all_repoagents = ["checksum"] - all_caches = ["local", "redis"] - all_filesystems = ["gcs", "s3", "azure_storage"] - all_endpoints = ["http", "grpc", "sagemaker", "vertex-ai"] - - FLAGS.enable_logging = True - FLAGS.enable_stats = True - FLAGS.enable_metrics = True - FLAGS.enable_gpu_metrics = True - FLAGS.enable_cpu_metrics = True - FLAGS.enable_tracing = True - FLAGS.enable_nvtx = True - FLAGS.enable_gpu = True + if target_platform() != "windows": + all_backends = [ + "ensemble", + "identity", + "square", + "repeat", + "onnxruntime", + "python", + "dali", + "pytorch", + "openvino", + "fil", + "tensorrt", + ] + all_repoagents = ["checksum"] + all_caches = ["local", "redis"] + all_filesystems = ["gcs", "s3", "azure_storage"] + all_endpoints = ["http", "grpc", "sagemaker", "vertex-ai"] + + FLAGS.enable_logging = True + FLAGS.enable_stats = True + FLAGS.enable_metrics = True + FLAGS.enable_gpu_metrics = True + FLAGS.enable_cpu_metrics = True + FLAGS.enable_tracing = True + FLAGS.enable_nvtx = True + FLAGS.enable_gpu = True + else: + all_backends = [ + "ensemble", + "identity", + "square", + "repeat", + "onnxruntime", + "openvino", + "tensorrt", + ] + all_repoagents = ["checksum"] + all_caches = ["local", "redis"] + all_filesystems = [] + all_endpoints = ["http", "grpc"] + + FLAGS.enable_logging = True + FLAGS.enable_stats = True + FLAGS.enable_tracing = True + FLAGS.enable_gpu = True requested_backends = [] for be in FLAGS.backend: @@ -2230,11 +2495,17 @@ def enable_all(): required=False, help="Do not use Docker --pull argument when building container.", ) + parser.add_argument( + "--container-memory", + default=None, + required=False, + help="Value for Docker --memory argument. Used only for windows builds.", + ) parser.add_argument( "--target-platform", required=False, default=None, - help='Target platform for build, can be "linux", "rhel" or "igpu". If not specified, build targets the current platform.', + help='Target platform for build, can be "linux", "rhel", "windows" or "igpu". If not specified, build targets the current platform.', ) parser.add_argument( "--target-machine", @@ -2386,6 +2657,20 @@ def enable_all(): required=False, help="Enable ARM MALI GPU support.", ) + parser.add_argument( + "--enable-mysql-odbc", + action="store_true", + required=False, + help="Build tritonserver with MySQL ODBC connection pool. For host builds install unixodbc-dev (Debian/Ubuntu) or unixODBC-devel (RHEL). Container builds add these when this flag is set.", + ) + parser.add_argument( + "--pubmatic-bt-tritonserver-image", + action="store_true", + required=False, + help='After building image "tritonserver", also build "pubmatic-bt-tritonserver" ' + "(Dockerfile.pubmatic-bt-tritonserver: ODBC driver + replace-artifacts/odbc.ini and " + "replace-artifacts/triton-dmconfig.json). Ubuntu/Debian-based container builds only.", + ) parser.add_argument( "--min-compute-capability", type=str, @@ -2508,6 +2793,12 @@ def enable_all(): default=DEFAULT_TRITON_VERSION_MAP["dcgm_version"], help="This flag sets the DCGM version for Triton Inference Server to be built. Default: the latest supported version.", ) + parser.add_argument( + "--vllm-version", + required=False, + default=DEFAULT_TRITON_VERSION_MAP["vllm_version"], + help="This flag sets the vLLM version for Triton Inference Server to be built. Default: the latest supported version.", + ) parser.add_argument( "--rhel-py-version", required=False, @@ -2522,16 +2813,11 @@ def enable_all(): metavar=("key", "value"), help="Add build secrets in the form of . These secrets are used during the build process for vllm. The secrets are passed to the Docker build step as `--secret id=`. The following keys are expected and their purposes are described below:\n\n" " - 'req': A file containing a list of dependencies for pip (e.g., requirements.txt).\n" + " - 'vllm_index_url': The index URL for the pip install.\n" + " - 'pytorch_triton_url': The location of the PyTorch wheel to download.\n" " - 'build_public_vllm': A flag (default is 'true') indicating whether to build the public VLLM version.\n\n" "Ensure that the required environment variables for these secrets are set before running the build.", ) - parser.add_argument( - "--triton-wheels-dependencies-group", - required=False, - type=str, - default="all", - help="The group of dependencies for Triton wheels to be installed. Default value is 'all'.", - ) FLAGS = parser.parse_args() if FLAGS.image is None: @@ -2561,13 +2847,6 @@ def enable_all(): if FLAGS.build_secret is None: FLAGS.build_secret = [] - FLAGS.boost_url = os.getenv( - "TRITON_BOOST_URL", - "https://archives.boost.io/release/1.80.0/source/boost_1_80_0.tar.gz", - ) - FLAGS.boost_sha256 = ( - "4b2136f98bdd1f5857f1c3dea9ac2018effe65286cf251534b6ae20cc45e1847" - ) # if --enable-all is specified, then update FLAGS to enable all # settings, backends, repo-agents, caches, file systems, endpoints, etc. if FLAGS.enable_all: @@ -2614,11 +2893,12 @@ def enable_all(): # backends, repo-agents, and caches if a repo-tag is not given # explicitly. For release branches we use the release branch as # the default, otherwise we use 'main'. - default_repo_tag = ( - "main" - if FLAGS.triton_container_version.endswith("dev") - else "r" + FLAGS.triton_container_version - ) + default_repo_tag = "main" + cver = FLAGS.upstream_container_version + if cver is None: + cver = FLAGS.triton_container_version + if not cver.endswith("dev"): + default_repo_tag = "r" + cver log("default repo-tag: {}".format(default_repo_tag)) # For other versions use the TRITON_VERSION_MAP unless explicitly @@ -2656,6 +2936,8 @@ def enable_all(): secrets = dict(getattr(FLAGS, "build_secret", [])) if secrets: requirements = secrets.get("req", "") + vllm_index_url = secrets.get("vllm_index_url", "") + pytorch_triton_url = secrets.get("pytorch_triton_url", "") build_public_vllm = secrets.get("build_public_vllm", "true") log('Build Arg for BUILD_PUBLIC_VLLM: "{}"'.format(build_public_vllm)) @@ -2685,7 +2967,7 @@ def enable_all(): len(parts) != 2, "--image must specify ," ) fail_if( - parts[0] not in ["base", "gpu-base", "pytorch", "inference"], + parts[0] not in ["base", "gpu-base", "pytorch"], "unsupported value for --image", ) log('image "{}": "{}"'.format(parts[0], parts[1])) @@ -2788,12 +3070,21 @@ def enable_all(): script_install_dir = script_ci_dir = FLAGS.install_dir script_cmake_dir = FLAGS.cmake_dir if not FLAGS.no_container_build: - script_build_dir = os.path.normpath(os.path.join(FLAGS.tmp_dir, "tritonbuild")) + # FLAGS.tmp_dir may be specified with "\" on Windows, adjust + # to "/" for docker usage. + script_build_dir = os.path.normpath( + os.path.join(FLAGS.tmp_dir, "tritonbuild").replace("\\", "/") + ) script_install_dir = os.path.normpath(os.path.join(script_build_dir, "install")) script_ci_dir = os.path.normpath(os.path.join(script_build_dir, "ci")) - script_repo_dir = script_cmake_dir = "/workspace" + if target_platform() == "windows": + script_repo_dir = script_cmake_dir = os.path.normpath("c:/workspace") + else: + script_repo_dir = script_cmake_dir = "/workspace" script_name = "cmake_build" + if target_platform() == "windows": + script_name += ".ps1" # Write the build script that invokes cmake for the core, backends, repo-agents, and caches. pathlib.Path(FLAGS.build_dir).mkdir(parents=True, exist_ok=True) @@ -2894,7 +3185,8 @@ def enable_all(): # written to the build-dir while running the docker container # may have root ownership, so give them permissions to be # managed by all users on the host system. - finalize_build(cmake_script, script_install_dir, script_ci_dir) + if target_platform() != "windows": + finalize_build(cmake_script, script_install_dir, script_ci_dir) # If --no-container-build is not specified then we perform the # actual build within a docker container and from that create the @@ -2903,6 +3195,8 @@ def enable_all(): # the build process. if not FLAGS.no_container_build: script_name = "docker_build" + if target_platform() == "windows": + script_name += ".ps1" create_build_dockerfiles( script_build_dir, images, backends, repoagents, caches, FLAGS.endpoint @@ -2913,6 +3207,12 @@ def enable_all(): # container-based build is requested use 'docker_build' script, # otherwise build directly on this system using cmake script. if not FLAGS.dryrun: - p = subprocess.Popen([f"./{script_name}"], cwd=FLAGS.build_dir) + if target_platform() == "windows": + p = subprocess.Popen( + ["powershell.exe", "-noexit", "-File", f"./{script_name}"], + cwd=FLAGS.build_dir, + ) + else: + p = subprocess.Popen([f"./{script_name}"], cwd=FLAGS.build_dir) p.wait() fail_if(p.returncode != 0, "build failed") diff --git a/compose.py b/compose.py index ef1342c0fe..88c6b15bf3 100755 --- a/compose.py +++ b/compose.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -98,12 +98,12 @@ def start_dockerfile(ddir, images, argmap, dockerfile_name, backends): # Copy over files df += """ WORKDIR /opt/tritonserver -COPY --from=full /opt/tritonserver/LICENSE . -COPY --from=full /opt/tritonserver/TRITON_VERSION . -COPY --from=full /opt/tritonserver/NVIDIA_Deep_Learning_Container_License.pdf . -COPY --from=full /opt/tritonserver/bin bin/ -COPY --from=full /opt/tritonserver/lib lib/ -COPY --from=full /opt/tritonserver/include include/ +COPY --chown=1000:1000 --from=full /opt/tritonserver/LICENSE . +COPY --chown=1000:1000 --from=full /opt/tritonserver/TRITON_VERSION . +COPY --chown=1000:1000 --from=full /opt/tritonserver/NVIDIA_Deep_Learning_Container_License.pdf . +COPY --chown=1000:1000 --from=full /opt/tritonserver/bin bin/ +COPY --chown=1000:1000 --from=full /opt/tritonserver/lib lib/ +COPY --chown=1000:1000 --from=full /opt/tritonserver/include include/ """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) @@ -112,10 +112,15 @@ def start_dockerfile(ddir, images, argmap, dockerfile_name, backends): def add_requested_backends(ddir, dockerfile_name, backends): df = "# Copying over backends \n" for backend in backends: - df += """COPY --from=full /opt/tritonserver/backends/{} /opt/tritonserver/backends/{} + df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/backends/{} /opt/tritonserver/backends/{} """.format( backend, backend ) + if len(backends) > 0: + df += """ +# Top-level /opt/tritonserver/backends not copied so need to explicitly set permissions here +RUN chown triton-server:triton-server /opt/tritonserver/backends +""" with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) @@ -123,10 +128,15 @@ def add_requested_backends(ddir, dockerfile_name, backends): def add_requested_repoagents(ddir, dockerfile_name, repoagents): df = "# Copying over repoagents \n" for ra in repoagents: - df += """COPY --from=full /opt/tritonserver/repoagents/{} /opt/tritonserver/repoagents/{} + df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/repoagents/{} /opt/tritonserver/repoagents/{} """.format( ra, ra ) + if len(repoagents) > 0: + df += """ +# Top-level /opt/tritonserver/repoagents not copied so need to explicitly set permissions here +RUN chown triton-server:triton-server /opt/tritonserver/repoagents +""" with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) @@ -134,10 +144,15 @@ def add_requested_repoagents(ddir, dockerfile_name, repoagents): def add_requested_caches(ddir, dockerfile_name, caches): df = "# Copying over caches \n" for cache in caches: - df += """COPY --from=full /opt/tritonserver/caches/{} /opt/tritonserver/caches/{} + df += """COPY --chown=1000:1000 --from=full /opt/tritonserver/caches/{} /opt/tritonserver/caches/{} """.format( cache, cache ) + if len(caches) > 0: + df += """ +# Top-level /opt/tritonserver/caches not copied so need to explicitly set permissions here +RUN chown triton-server:triton-server /opt/tritonserver/caches +""" with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) @@ -148,7 +163,7 @@ def end_dockerfile(ddir, dockerfile_name, argmap): if argmap["SAGEMAKER_ENDPOINT"]: df += """ LABEL com.amazonaws.sagemaker.capabilities.accept-bind-to-port=true -COPY --from=full /usr/bin/serve /usr/bin/. +COPY --chown=1000:1000 --from=full /usr/bin/serve /usr/bin/. """ with open(os.path.join(ddir, dockerfile_name), "a") as dfile: dfile.write(df) @@ -283,7 +298,7 @@ def create_argmap(images, skip_pull): dcgm_ver = re.search("DCGM_VERSION=([\S]{4,}) ", vars) dcgm_version = "" if dcgm_ver is None: - dcgm_version = "4.5.3-1" + dcgm_version = "3.3.6" log( "WARNING: DCGM version not found from image, installing the earlierst version {}".format( dcgm_version diff --git a/config/database_config.sample.json b/config/database_config.sample.json new file mode 100644 index 0000000000..db12e70189 --- /dev/null +++ b/config/database_config.sample.json @@ -0,0 +1,13 @@ +{ + "databaseIp" : "10.0.0.1", + "databasePort" : 3306, + "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver", + "primaryDSNName" : "PrimaryCentralisedMySQL", + "secondaryDSNName" : "SecondaryCentralisedMySQL", + "dsnUserName" : "kdbuser", + "dsnUserPassword" : "REPLACE_WITH_SECRET", + "queryRetryCount": 3, + "dcId": 1, + "minPoolConnections": 2, + "maxPoolConnections": 5 +} diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index c94f832aa8..d7b424435c 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.03-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index bd360e7955..481c4bd44c 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.69.0 +appVersion: "2.56.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index b911db4afd..a8458c97dc 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.03-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r26.05/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.03/docs/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r26.05/README.md + # see https://github.com/triton-inference-server/server/blob/r25.03/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index 9784c9d252..eb0a18b32f 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:25.03-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 0e1347f4fd..1671b334c8 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -1,4 +1,4 @@ -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:26.05-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.03-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 4b4468d89d..31af7064f5 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.67 -export MINOR_VERSION=2.69.0 -export NGC_VERSION=26.05-py3 +export MAJOR_VERSION=2.56 +export MINOR_VERSION=2.56.0 +export NGC_VERSION=25.03-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index d150f0e8d7..1d490f7c9c 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. apiVersion: v1 -appVersion: "2.68" +appVersion: "2.56" description: Triton Inference Server name: triton-inference-server -version: 2.69.0 +version: 2.56.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 362107e71a..d97d89f04d 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/26.05 -publishedVersion: '2.69.0' +modelRepositoryPath: gs://triton_sample_models/25.03 +publishedVersion: '2.56.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 26.05-py3 + tag: 25.03-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 4c312c9880..7c5f5cc425 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.56.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -88,11 +88,11 @@ properties: default: 85 modelRepositoryPath: type: string - title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, Pytorch, Onnx ... etc. + title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. default: gs://triton_sample_models/models image.ldPreloadPath: type: string - title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin or custom ops, the compiled shared library must be provided via LD_PRELOAD environment variable. + title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin or Tensorflow custom ops, the compiled shared library must be provided via LD_PRELOAD environment variable. default: '' image.logVerboseLevel: type: integer diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index ccf3b157c4..2569628502 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.56.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,10 +89,10 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/26.05 + default: gs://triton_sample_models/25.03 image.ldPreloadPath: type: string - title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. + title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin or Tensorflow custom ops, the compiled shared library must be provided via LD_PRELOAD environment variable. default: '' image.logVerboseLevel: type: integer diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index fff7466da4..8b3e284fdd 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -1,5 +1,5 @@ +# OpenAI-Compatible Frontend for Triton Inference Server (Beta) + +> [!NOTE] +> The OpenAI-Compatible API is currently in BETA. Its features and functionality +> are subject to change as we collect feedback. We're excited to hear any thoughts +> you have and what features you'd like to see! + +## Pre-requisites + +1. Docker + NVIDIA Container Runtime +2. A correctly configured `HF_TOKEN` for access to HuggingFace models. + - The current examples and testing primarily use the + [`meta-llama/Meta-Llama-3.1-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) + model, but you can manually bring your own models and adjust accordingly. + +## VLLM + +1. Launch the container and install dependencies: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 +``` + +2. Launch the OpenAI-compatible Triton Inference Server: +```bash +cd /opt/tritonserver/python/openai + +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository tests/vllm_models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +
+Example output + +``` +... ++-----------------------+---------+--------+ +| Model | Version | Status | ++-----------------------+---------+--------+ +| llama-3.1-8b-instruct | 1 | READY | <- Correct Model Loaded in Triton ++-----------------------+---------+--------+ +... +Found model: name='llama-3.1-8b-instruct', backend='vllm' +[WARNING] Adding CORS for the following origins: ['http://localhost'] +INFO: Started server process [126] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully +``` + +
+ +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-6930b296-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": + { + "content": "This is only a test.", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1727679085, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +4. Send a `/v1/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "prompt": "Machine learning is" +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-d51df75c-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "text": " a field of computer science that focuses on developing algorithms that allow computers to learn from" + } + ], + "created": 1727679266, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "text_completion", + "usage": null +} +``` + +
+ +5. Benchmark with `genai-perf`: +- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) +- Or try using genai-perf from the [SDK container](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) + +```bash +MODEL="llama-3.1-8b-instruct" +TOKENIZER="meta-llama/Meta-Llama-3.1-8B-Instruct" +genai-perf profile \ + --model ${MODEL} \ + --tokenizer ${TOKENIZER} \ + --service-kind openai \ + --endpoint-type chat \ + --url localhost:9000 \ + --streaming +``` + +
+Example output + +``` +2024-10-14 22:43 [INFO] genai_perf.parser:82 - Profiling these models: llama-3.1-8b-instruct +2024-10-14 22:43 [INFO] genai_perf.wrapper:163 - Running Perf Analyzer : 'perf_analyzer -m llama-3.1-8b-instruct --async --input-data artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/inputs.json -i http --concurrency-range 1 --endpoint v1/chat/completions --service-kind openai -u localhost:9000 --measurement-interval 10000 --stability-percentage 999 --profile-export-file artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export.json' + NVIDIA GenAI-Perf | LLM Metrics +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ +┃ Statistic ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p75 ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ +│ Time to first token (ms) │ 71.66 │ 64.32 │ 86.52 │ 76.13 │ 74.92 │ 73.26 │ +│ Inter token latency (ms) │ 18.47 │ 18.25 │ 18.72 │ 18.67 │ 18.61 │ 18.53 │ +│ Request latency (ms) │ 348.00 │ 274.60 │ 362.27 │ 355.41 │ 352.29 │ 350.66 │ +│ Output sequence length │ 15.96 │ 12.00 │ 16.00 │ 16.00 │ 16.00 │ 16.00 │ +│ Input sequence length │ 549.66 │ 548.00 │ 551.00 │ 550.00 │ 550.00 │ 550.00 │ +│ Output token throughput (per sec) │ 45.84 │ N/A │ N/A │ N/A │ N/A │ N/A │ +│ Request throughput (per sec) │ 2.87 │ N/A │ N/A │ N/A │ N/A │ N/A │ +└───────────────────────────────────┴────────┴────────┴────────┴────────┴────────┴────────┘ +2024-10-14 22:44 [INFO] genai_perf.export_data.json_exporter:62 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.json +2024-10-14 22:44 [INFO] genai_perf.export_data.csv_exporter:71 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.csv +``` + +
+ +6. Use the OpenAI python client directly: +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:9000/v1", + api_key="EMPTY", +) + +model = "llama-3.1-8b-instruct" +completion = client.chat.completions.create( + model=model, + messages=[ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "What are LLMs?"}, + ], + max_tokens=256, +) + +print(completion.choices[0].message.content) +``` + +7. Run tests (NOTE: The server should not be running, the tests will handle starting/stopping the server as necessary): +```bash +cd /opt/tritonserver/python/openai/ +pip install -r requirements-test.txt + +pytest -v tests/ +``` + +## TensorRT-LLM + +0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: + - [Triton CLI](https://github.com/triton-inference-server/triton_cli/) + - [TRT-LLM Backend Quickstart](https://github.com/triton-inference-server/tensorrtllm_backend?tab=readme-ov-file#quick-start) + +1. Launch the container: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + -e TRTLLM_ORCHESTRATOR=1 \ + nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 +``` + +2. Install dependencies inside the container: +```bash +# Install python bindings for tritonserver and tritonfrontend +pip install /opt/tritonserver/python/triton*.whl + +# Install application requirements +git clone https://github.com/triton-inference-server/server.git +cd server/python/openai/ +pip install -r requirements.txt +``` + +2. Launch the OpenAI server: +```bash +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository path/to/models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +# MODEL should be the client-facing model name in your model repository for a pipeline like TRT-LLM. +# For example, this could also be "ensemble", or something like "gpt2" if generated from Triton CLI +MODEL="tensorrt_llm_bls" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-704c758c-8a84-11ef-b106-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "It looks like you're testing the system!", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1728948689, + "model": "llama-3-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +The other examples should be the same as vLLM, except that you should set `MODEL="tensorrt_llm_bls"` or `MODEL="ensemble"`, +everywhere applicable as seen in the example request above. + +## KServe Frontends + +To support serving requests through both the OpenAI-Compatible and +KServe Predict v2 frontends to the same running Triton Inference Server, +the `tritonfrontend` python bindings are included for optional use in this +application as well. + +You can opt-in to including these additional frontends, assuming `tritonfrontend` +is installed, with `--enable-kserve-frontends` like below: + +``` +python3 openai_frontend/main.py \ + --model-repository tests/vllm_models \ + --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ + --enable-kserve-frontends +``` + +See `python3 openai_frontend/main.py --help` for more information on the +available arguments and default values. + +For more information on the `tritonfrontend` python bindings, see the docs +[here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md). + +## Model Parallelism Support + +- [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend)) + - ex: Configure `tensor_parallel_size: 2` in the + [model.json](https://github.com/triton-inference-server/vllm_backend/blob/main/samples/model_repository/vllm_model/1/model.json) +- [x] TensorRT-LLM ([Orchestrator Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#orchestrator-mode)) + - Set the following environment variable: `export TRTLLM_ORCHESTRATOR=1` +- [ ] TensorRT-LLM ([Leader Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#leader-mode)) + - Not currently supported diff --git a/docs/client_guide/python.rst b/docs/client_guide/python.rst index e546f7b18e..545f4f6042 100644 --- a/docs/client_guide/python.rst +++ b/docs/client_guide/python.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -29,10 +29,11 @@ Python #### +.. include:: python_readme.rst + .. toctree:: :maxdepth: 1 :hidden: - Overview <../tutorials/Triton_Inference_Server_Python_API/README.md> Kafka I/O <../tutorials/Triton_Inference_Server_Python_API/examples/kafka-io/README.md> Rayserve <../tutorials/Triton_Inference_Server_Python_API/examples/rayserve/README.md> \ No newline at end of file diff --git a/docs/client_guide/python_readme.rst b/docs/client_guide/python_readme.rst new file mode 100644 index 0000000000..e7a79abe60 --- /dev/null +++ b/docs/client_guide/python_readme.rst @@ -0,0 +1,268 @@ +.. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +Triton Inference Server In-Process Python API [BETA] +==================================================== + +Starting with release 24.01 Triton Inference Server will include a +Python package enabling developers to embed Triton Inference Server +instances in their Python applications. The in-process Python API is +designed to match the functionality of the in-process C API while +providing a higher level abstraction. At its core the API relies on a +1:1 python binding of the C API and provides all the flexibility and +power of the C API with a simpler to use interface. + + [!Note] As the API is in BETA please expect some changes as we test + out different features and get feedback. All feedback is weclome and + we look forward to hearing from you! + +| `Requirements <#requirements>`__ \| `Installation <#installation>`__ + \| `Hello World <#hello-world>`__ \| `Stable + Diffusion <#stable-diffusion>`__ \| `Ray Serve + Deployment <../tutorials/Triton_Inference_Server_Python_API/examples/rayserve>`__ \| +Requirements +------------ + +The following instructions require a linux system with Docker installed. +For CUDA support, make sure your CUDA driver meets the requirements in +“NVIDIA Driver” section of Deep Learning Framework support matrix: +https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html + +Installation +------------ + +The tutorial and Python API package are designed to be installed and run +within the ``nvcr.io/nvidia/tritonserver:24.01-py3`` docker image. + +A set of convenience scripts are provided to create a docker image based +on the ``nvcr.io/nvidia/tritonserver:24.01-py3`` image with the Python +API installed plus additional dependencies required for the examples. + +Triton Inference Server 24.01 + Python API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Clone Repository +^^^^^^^^^^^^^^^^ + +.. code:: bash + git clone https://github.com/triton-inference-server/tutorials.git + cd tutorials/Triton_Inference_Server_Python_API +Build ``triton-python-api:r24.01`` Image +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code:: bash + ./build.sh +Supported Backends +^^^^^^^^^^^^^^^^^^ + +The built image includes all the backends shipped by default in the +tritonserver ``nvcr.io/nvidia/tritonserver:24.01-py3`` container. + +:: + + dali fil identity onnxruntime openvino python pytorch repeat square tensorflow tensorrt + +Included Models +^^^^^^^^^^^^^^^ + +The ``default`` build includes an ``identity`` model that can be used +for exercising basic operations including sending input tensors of +different data types. The ``identity`` model copies provided inputs of +``shape [-1, -1]`` to outputs of shape ``[-1, -1]``. Inputs are named +``data_type_input`` and outputs are named ``data_type_output`` +(e.g. ``string_input``, ``string_output``, ``fp16_input``, +``fp16_output``). + +Hello World +----------- + +Start ``triton-python-api:r24.01`` Container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following command starts a container and volume mounts the current +directory as ``workspace``. + +.. code:: bash + ./run.sh +Enter Python Shell +~~~~~~~~~~~~~~~~~~ + +.. code:: bash + python3 +Create and Start a Server Instance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + import tritonserver + server = tritonserver.Server(model_repository="/workspace/identity-models") + server.start() +List Models +~~~~~~~~~~~ + +:: + + server.models() + +Example Output +^^^^^^^^^^^^^^ + +``server.models()`` returns a dictionary of the available models with +their current state. + +.. code:: python + {('identity', 1): {'name': 'identity', 'version': 1, 'state': 'READY'}} +Send an Inference Request +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + model = server.model("identity") + responses = model.infer(inputs={"string_input":[["hello world!"]]}) +Iterate through Responses +~~~~~~~~~~~~~~~~~~~~~~~~~ + +``model.infer()`` returns an iterator that can be used to process the +results of an inference request. + +.. code:: python + for response in responses: + print(response.outputs["string_output"].to_string_array()) +.. _example-output-1: + +Example Output +^^^^^^^^^^^^^^ + +.. code:: python + [['hello world!']] +Stable Diffusion +---------------- + +This example is based on the +`Popular_Models_Guide/StableDiffusion <../tutorials/Popular_Models_Guide/StableDiffusion/README.html>`__ +tutorial. + +Build ``triton-python-api:r24.01-diffusion`` Image and Stable Diffusion Models +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Please note the following command will take many minutes depending on +your hardware configuration and network connection. + +.. code:: bash + ./build.sh --framework diffusion --build-models +.. _supported-backends-1: + +Supported Backends +^^^^^^^^^^^^^^^^^^ + +The built image includes all the backends shipped by default in the +tritonserver ``nvcr.io/nvidia/tritonserver:24.01-py3`` container. + +:: + + dali fil identity onnxruntime openvino python pytorch repeat square tensorflow tensorrt + +.. _included-models-1: + +Included Models +^^^^^^^^^^^^^^^ + +The ``diffusion`` build includes a ``stable_diffustion`` pipeline that +takes a text prompt and returns a generated image. For more details on +the models and pipeline please see the +`Popular_Models_Guide/StableDiffusion <../tutorials/Popular_Models_Guide/StableDiffusion/README.html>`__ +tutorial. + +Start Container +~~~~~~~~~~~~~~~ + +The following command starts a container and volume mounts the current +directory as ``workspace``. + +.. code:: bash + ./run.sh --framework diffusion +.. _enter-python-shell-1: + +Enter Python Shell +~~~~~~~~~~~~~~~~~~ + +.. code:: bash + python3 +.. _create-and-start-a-server-instance-1: + +Create and Start a Server Instance +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + import tritonserver + import numpy + from PIL import Image + server = tritonserver.Server(model_repository="/workspace/diffusion-models") + server.start() +.. _list-models-1: + +List Models +~~~~~~~~~~~ + +:: + + server.models() + +.. _example-output-2: + +Example Output +^^^^^^^^^^^^^^ + +.. code:: python + {('stable_diffusion', 1): {'name': 'stable_diffusion', 'version': 1, 'state': 'READY'}, ('text_encoder', 1): {'name': 'text_encoder', 'version': 1, 'state': 'READY'}, ('vae', 1): {'name': 'vae', 'version': 1, 'state': 'READY'}} +.. _send-an-inference-request-1: + +Send an Inference Request +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + model = server.model("stable_diffusion") + responses = model.infer(inputs={"prompt":[["butterfly in new york, realistic, 4k, photograph"]]}) +Iterate through Responses and save image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + for response in responses: + generated_image = numpy.from_dlpack(response.outputs["generated_image"]) + generated_image = generated_image.squeeze().astype(numpy.uint8) + image_ = Image.fromarray(generated_image) + image_.save("sample_generated_image.jpg") +.. _example-output-3: + +Example Output +^^^^^^^^^^^^^^ + +.. figure:: ../tutorials/Triton_Inference_Server_Python_API/docs/sample_generated_image.jpg + :alt: sample_generated_image + + sample_generated_image \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 387e0f3c37..0b44f7c8b2 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -35,12 +35,9 @@ # -- Path setup -------------------------------------------------------------- import json -import logging import os import re -import subprocess from datetime import date -from logging.handlers import RotatingFileHandler # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -64,45 +61,6 @@ # at the end of the file. # current_dir = os.getcwd() # os.chdir("docs") -# -- Setup logger ------------------------------------------------------------ - - -def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_count=5): - logger = logging.getLogger(name) - logger.setLevel(level) - - # Prevent adding multiple handlers if the function is called multiple times - if not logger.handlers: - # Create handlers - file_handler = RotatingFileHandler( - log_file, maxBytes=max_bytes, backupCount=backup_count - ) - console_handler = logging.StreamHandler() - - # Set the logging level for handlers - file_handler.setLevel(level) - console_handler.setLevel(level) - - # Create a logging format - BLUE = "\033[94m" - RESET = "\033[0m" - formatter = logging.Formatter( - f"{BLUE}%(asctime)s - %(name)s - %(levelname)s - {RESET}%(message)s" - ) - file_handler.setFormatter(formatter) - console_handler.setFormatter(formatter) - - # Add handlers to the logger - logger.addHandler(file_handler) - logger.addHandler(console_handler) - return logger - - -logger = setup_logger( - os.path.basename(__file__), - os.environ.get("TRITON_SERVER_DOCS_LOG_FILE", "/tmp/docs.log"), -) -logger.info(f"Defined logger for {os.path.basename(__file__)}") # -- Project information ----------------------------------------------------- @@ -112,19 +70,14 @@ def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_c # Get the version of Triton this is building. version_long = "0.0.0" -logger.info(f"Getting version from ../TRITON_VERSION") with open("../TRITON_VERSION") as f: version_long = f.readline() version_long = version_long.strip() - logger.info(f"Version: {version_long}") - version_short = re.match(r"^[\d]+\.[\d]+\.[\d]+", version_long).group(0) -logger.info(f"Version short: {version_short}") version_short_split = version_short.split(".") -logger.info(f"Version short split: {version_short_split}") one_before = f"{version_short_split[0]}.{int(version_short_split[1]) - 1}.{version_short_split[2]}" -logger.info(f"One before: {one_before}") + # maintain left-side bar toctrees in `contents` file # so it doesn't show up needlessly in the index page @@ -227,7 +180,7 @@ def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_c "json_url": "https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/_static/switcher.json", "version_match": one_before if "dev" in version_long else version_short, }, - # "navbar_start": ["navbar-logo", "version-switcher"], + "navbar_start": ["navbar-logo", "version-switcher"], "primary_sidebar_end": [], } @@ -241,8 +194,6 @@ def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_c } ) -logger.info(f"html_theme_options: {html_theme_options}") - deploy_ngc_org = "nvidia" deploy_ngc_team = "triton" myst_substitutions = { @@ -252,8 +203,6 @@ def setup_logger(name, log_file, level=logging.INFO, max_bytes=1048576, backup_c else deploy_ngc_org, } -logger.info(f"myst_substitutions: {myst_substitutions}") - def ultimateReplace(app, docname, source): result = source[0] @@ -270,7 +219,6 @@ def ultimateReplace(app, docname, source): if deploy_ngc_team else deploy_ngc_org, } -logger.info(f"ultimate_replacements: {ultimate_replacements}") # bibtex_bibfiles = ["references.bib"] # To test that style looks good with common bibtex config @@ -285,24 +233,25 @@ def ultimateReplace(app, docname, source): # SETUP SWITCHER ############################### switcher_path = os.path.join(html_static_path[0], "switcher.json") -logger.info(f"switcher_path: {switcher_path}") versions = [] +# Triton 2 releases +correction = -1 if "dev" in version_long else 0 +upper_bound = version_short.split(".")[1] +for i in range(2, int(version_short.split(".")[1]) + correction): + versions.append((f"2.{i}.0", f"triton-inference-server-2{i}0")) -# Obtain Triton Server Release Tags. -tags = subprocess.run(["git", "tag", "--list", "v*"], capture_output=True, text=True) -tags_list = sorted(tags.stdout.strip().splitlines(), key=Version, reverse=True) -logger.info(f"Found source tags: {tags_list}") - -for v in tags_list: - versions.append( - ( - v.replace("v", ""), - f"triton-inference-server-{v.replace('v', '').replace('.', '')}", - ) - ) +# Triton 1 releases +for i in range(0, 15): + versions.append((f"1.{i}.0", f"tensorrt_inference_server_1{i}0")) + +# Triton Beta Releases +for i in range(1, 11): + versions.append((f"0.{i}.0_beta", f"inference_server_0{i}0_beta")) -logger.info(f"Defined dictionary of versions: {versions}") +# Patch releases +# Add here. +versions = sorted(versions, key=lambda v: Version(v[0]), reverse=True) # Build switcher data json_data = [] @@ -314,7 +263,6 @@ def ultimateReplace(app, docname, source): "url": f"https://docs.nvidia.com/deeplearning/triton-inference-server/archives/{v[1]}/user-guide/docs", } ) - if "dev" in version_long: json_data.insert( 0, @@ -336,7 +284,6 @@ def ultimateReplace(app, docname, source): # Trim to last N releases. json_data = json_data[0:12] -logger.info(f"Trimmed to last 12 release...") json_data.append( { @@ -346,22 +293,19 @@ def ultimateReplace(app, docname, source): } ) +# validate the links for i, d in enumerate(json_data): - logger.info(f"Validating link: {d['url']}") h = httplib2.Http() resp = h.request(d["url"], "HEAD") if int(resp[0]["status"]) >= 400: print(d["url"], "NOK", resp[0]["status"]) - # exit(1) + exit(1) -logger.info(f"Writing switcher data to file: {switcher_path}") +# Write switcher data to file with open(switcher_path, "w") as f: json.dump(json_data, f, ensure_ascii=False, indent=4) -logger.info("Configuration completed...") - - def setup(app): app.add_config_value("ultimate_replacements", {}, True) app.connect("source-read", ultimateReplace) diff --git a/docs/contents.rst b/docs/contents.rst index 420293aa76..dfff933e31 100644 --- a/docs/contents.rst +++ b/docs/contents.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -36,8 +36,8 @@ :hidden: :caption: Getting Started - getting_started/quick_deployment - LLM With TensorRT-LLM + getting_started/quick_deployment_by_backend + LLM With TRT-LLM Multimodal model <../tutorials/Popular_Models_Guide/Llava1.5/llava_trtllm_guide.md> Stable diffusion <../tutorials/Popular_Models_Guide/StableDiffusion/README.md> @@ -54,7 +54,7 @@ Constrained Decoding <../tutorials/Feature_Guide/Constrained_Decoding/README.md> Function Calling <../tutorials/Feature_Guide/Function_Calling/README.md> - llm_features/speculative_decoding + llm_features/speculative_decoding_by_backend_type .. toctree:: :hidden: @@ -96,11 +96,12 @@ :hidden: :caption: Backends - TensorRT-LLM + TRT-LLM vLLM Python - PyTorch + Pytorch ONNX Runtime + TensorFlow TensorRT FIL DALI @@ -108,7 +109,7 @@ .. toctree:: :hidden: - :caption: Performance benchmarking and tuning + :caption: Perf benchmarking and tuning GenAI Perf Analyzer Performance Analyzer diff --git a/docs/customization_guide/build.md b/docs/customization_guide/build.md index 48495c7211..478f804bd4 100644 --- a/docs/customization_guide/build.md +++ b/docs/customization_guide/build.md @@ -1,5 +1,5 @@ - export RELEASE="26.05" + export RELEASE="24.07" docker run -it --net=host --gpus '"device=0"' nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk 17. ## Download the Phi-3 tokenizer @@ -354,7 +354,7 @@ All config files inside /tensorrtllm\_backend/all\_models/inflight\_batcher\_llm
ensemble/config.pbtxt - # Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -864,7 +864,7 @@ All config files inside /tensorrtllm\_backend/all\_models/inflight\_batcher\_llm
postprocessing/config.pbtxt - # Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -993,7 +993,7 @@ All config files inside /tensorrtllm\_backend/all\_models/inflight\_batcher\_llm
preprocessing/config.pbtxt - # Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -1188,7 +1188,7 @@ All config files inside /tensorrtllm\_backend/all\_models/inflight\_batcher\_llm tensorrt_llm/config.pbtxt - # Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/docs/getting_started/quick_deployment.rst b/docs/getting_started/quick_deployment_by_backend.rst similarity index 92% rename from docs/getting_started/quick_deployment.rst rename to docs/getting_started/quick_deployment_by_backend.rst index b20775684a..aefa56787b 100644 --- a/docs/getting_started/quick_deployment.rst +++ b/docs/getting_started/quick_deployment_by_backend.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -29,14 +29,16 @@ Quick Deployment Guide by backend #### +.. include:: quick_start.rst + .. toctree:: :maxdepth: 1 :hidden: - Quickstart TRT-LLM vLLM <../tutorials/Popular_Models_Guide/Llama2/vllm_guide.md> Python with HuggingFace <../tutorials/Quick_Deploy/HuggingFaceTransformers/README.md> PyTorch <../tutorials/Quick_Deploy/PyTorch/README.md> ONNX <../tutorials/Quick_Deploy/ONNX/README.md> + TensorFlow <../tutorials/Quick_Deploy/TensorFlow/README.md> Openvino <../tutorials/Quick_Deploy/OpenVINO/README.md> \ No newline at end of file diff --git a/docs/getting_started/quick_start.rst b/docs/getting_started/quick_start.rst new file mode 100644 index 0000000000..27f100e3cd --- /dev/null +++ b/docs/getting_started/quick_start.rst @@ -0,0 +1,175 @@ +.. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +Quickstart +========== + +**New to Triton Inference Server and want do just deploy your model +quickly?** Make use of `these +tutorials <../tutorials/README.html#quick-deploy>`__ to begin your Triton +journey! + +The Triton Inference Server is available as `buildable source +code <../customization_guide/build.html>`__, but the easiest way to +install and run Triton is to use the pre-built Docker image available +from the `NVIDIA GPU Cloud (NGC) `__. + +Launching and maintaining Triton Inference Server revolves around the +use of building model repositories. This tutorial will cover: + +- Creating a Model Repository +- Launching Triton +- Send an Inference Request + +Create A Model Repository +------------------------- + +The `model repository <../user_guide/model_repository.html>`__ is the +directory where you place the models that you want Triton to serve. An +example model repository is included in the +`docs/examples/model_repository `__. +Before using the repository, you must fetch any missing model definition +files from their public model zoos via the provided script. + +:: + + $ cd docs/examples + $ ./fetch_models.sh + +Launch Triton +------------- + +Triton is optimized to provide the best inferencing performance by using +GPUs, but it can also work on CPU-only systems. In both cases you can +use the same Triton Docker image. + +Run on System with GPUs +~~~~~~~~~~~~~~~~~~~~~~~ + +Use the following command to run Triton with the example model +repository you just created. The `NVIDIA Container +Toolkit `__ must be installed +for Docker to recognize the GPU(s). The –gpus=1 flag indicates that 1 +system GPU should be made available to Triton for inferencing. + +:: + + $ docker run --gpus=1 --rm -p8000:8000 -p8001:8001 -p8002:8002 -v/full/path/to/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:-py3 tritonserver --model-repository=/models + +Where is the version of Triton that you want to use (and pulled +above). After you start Triton you will see output on the console +showing the server starting up and loading the model. When you see +output like the following, Triton is ready to accept inference requests. + +:: + + +----------------------+---------+--------+ + | Model | Version | Status | + +----------------------+---------+--------+ + | | | READY | + | .. | . | .. | + | .. | . | .. | + +----------------------+---------+--------+ + ... + ... + ... + I1002 21:58:57.891440 62 grpc_server.cc:3914] Started GRPCInferenceService at 0.0.0.0:8001 + I1002 21:58:57.893177 62 http_server.cc:2717] Started HTTPService at 0.0.0.0:8000 + I1002 21:58:57.935518 62 http_server.cc:2736] Started Metrics Service at 0.0.0.0:8002 + +All the models should show “READY” status to indicate that they loaded +correctly. If a model fails to load the status will report the failure +and a reason for the failure. If your model is not displayed in the +table check the path to the model repository and your CUDA drivers. + +Run on CPU-Only System +~~~~~~~~~~~~~~~~~~~~~~ + +On a system without GPUs, Triton should be run without using the –gpus +flag to Docker, but is otherwise identical to what is described above. + +:: + + $ docker run --rm -p8000:8000 -p8001:8001 -p8002:8002 -v/full/path/to/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:-py3 tritonserver --model-repository=/models + +Because the –gpus flag is not used, a GPU is not available and Triton +will therefore be unable to load any model configuration that requires a +GPU. + +Verify Triton Is Running Correctly +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Use Triton’s *ready* endpoint to verify that the server and the models +are ready for inference. From the host system use curl to access the +HTTP endpoint that indicates server status. + +:: + + $ curl -v localhost:8000/v2/health/ready + ... + < HTTP/1.1 200 OK + < Content-Length: 0 + < Content-Type: text/plain + +The HTTP request returns status 200 if Triton is ready and non-200 if it +is not ready. + +Send an Inference Request +------------------------- + +Use docker pull to get the client libraries and examples image from NGC. + +:: + + $ docker pull nvcr.io/nvidia/tritonserver:-py3-sdk + +Where is the version that you want to pull. Run the client +image. + +:: + + $ docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:-py3-sdk + +From within the nvcr.io/nvidia/tritonserver:-py3-sdk image, run +the example image-client application to perform image classification +using the example densenet_onnx model. + +To send a request for the densenet_onnx model use an image from the +/workspace/images directory. In this case we ask for the top 3 +classifications. + +:: + + $ /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg + Request 0, batch size 1 + Image '/workspace/images/mug.jpg': + 15.346230 (504) = COFFEE MUG + 13.224326 (968) = CUP + 10.422965 (505) = COFFEEPOT \ No newline at end of file diff --git a/docs/getting_started/trtllm_user_guide.md b/docs/getting_started/trtllm_user_guide.md index a47d0c471d..7f128e98c7 100644 --- a/docs/getting_started/trtllm_user_guide.md +++ b/docs/getting_started/trtllm_user_guide.md @@ -1,5 +1,5 @@ -# [Triton Inference Server Release 26.05](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-26-05.html#rel-26-05) +# [Triton Inference Server Release 25.03](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-03.html#rel-25-03) -The Triton Inference Server container image, release 26.05, is available +The Triton Inference Server container image, release 25.03, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can -be found on the [GitHub Release Page](https://github.com/triton-inference-server/server/releases) +be found on the [GitHub Release Page](https://github.com/triton-inference-server/server/releases) \ No newline at end of file diff --git a/docs/llm_features/speculative_decoding.rst b/docs/llm_features/speculative_decoding.rst index 7e0df42bc5..debbcf52ae 100644 --- a/docs/llm_features/speculative_decoding.rst +++ b/docs/llm_features/speculative_decoding.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -25,14 +25,30 @@ .. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE .. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#### -Speculative Decoding -#### +.. raw:: html -.. toctree:: - :maxdepth: 1 - :hidden: - Overview <../tutorials/Feature_Guide/Speculative_Decoding/README.md> - TRT-LLM <../tutorials/Feature_Guide/Speculative_Decoding/TRT-LLM/README.md> - vLLM <../tutorials/Feature_Guide/Speculative_Decoding/vLLM/README.md> \ No newline at end of file +About Speculative Decoding +========================= +Speculative Decoding (also referred to as Speculative Sampling) is a set of techniques designed +to allow generation of more than one token per forward pass iteration. This can lead to a reduction +in the average per-token latency in situations where the GPU is underutilized due to small batch sizes. + +Speculative decoding involves predicting a sequence of future tokens, referred to as draft tokens, +using a method that is substantially more efficient than repeatedly executing the target Large Language +Model (LLM). These draft tokens are then collectively validated by processing them through the target LLM +in a single forward pass. The underlying assumptions are twofold: + +1. processing multiple draft tokens concurrently will be as rapid as processing a single token +2. multiple draft tokens will be validated successfully over the course of the full generation + +If the first assumption holds true, the latency of speculative decoding will no worse than the standard +approach. If the second holds, output token generation advances by statistically more than one token per +forward pass. The combination of both these allows speculative decoding to result in reduced latency. + +Performance Improvements +======================== +It's important to note that the effectiveness of speculative decoding techniques is highly dependent +on the specific task at hand. For instance, forecasting subsequent tokens in a code-completion scenario +may prove simpler than generating a summary for an article. `Spec-Bench `__ +shows the performance of different speculative decoding approaches on different tasks. \ No newline at end of file diff --git a/docs/llm_features/speculative_decoding_by_backend_type.rst b/docs/llm_features/speculative_decoding_by_backend_type.rst new file mode 100644 index 0000000000..a61f625626 --- /dev/null +++ b/docs/llm_features/speculative_decoding_by_backend_type.rst @@ -0,0 +1,39 @@ +.. +.. Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#### +Speculative Decoding +#### + +.. include:: speculative_decoding.rst + +.. toctree:: + :maxdepth: 1 + :hidden: + + TRT-LLM <../tutorials/Feature_Guide/Speculative_Decoding/TRT-LLM/README.md> + vLLM <../tutorials/Feature_Guide/Speculative_Decoding/vLLM/README.md> \ No newline at end of file diff --git a/docs/perf_benchmark/genai-perf-README.rst b/docs/perf_benchmark/genai-perf-README.rst new file mode 100644 index 0000000000..c4a3c7d73d --- /dev/null +++ b/docs/perf_benchmark/genai-perf-README.rst @@ -0,0 +1,686 @@ +.. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +GenAI-Perf +========== + +GenAI-Perf is a command line tool for measuring the throughput and +latency of generative AI models as served through an inference server. +For large language models (LLMs), GenAI-Perf provides metrics such as +`output token throughput <#output_token_throughput_metric>`__, `time to +first token <#time_to_first_token_metric>`__, `inter token +latency <#inter_token_latency_metric>`__, and `request +throughput <#request_throughput_metric>`__. For a full list of metrics +please see the `Metrics section <#metrics>`__. + +Users specify a model name, an inference server URL, the type of inputs +to use (synthetic or from dataset), and the type of load to generate +(number of concurrent requests, request rate). + +GenAI-Perf generates the specified load, measures the performance of the +inference server and reports the metrics in a simple table as console +output. The tool also logs all results in a csv and json file that can +be used to derive additional metrics and visualizations. The inference +server must already be running when GenAI-Perf is run. + +You can use GenAI-Perf to run performance benchmarks on - `Large +Language Models `__ - `Vision Language +Models `__ - `Embedding +Models `__ - `Ranking Models `__ - +`Multiple LoRA Adapters `__ + + [!Note] GenAI-Perf is currently in early release and under rapid + development. While we will try to remain consistent, command line + options and functionality are subject to change as the tool matures. + +.. raw:: html + + + +Installation +------------ + +The easiest way to install GenAI-Perf is through `Triton Server SDK +container `__. +Install the latest release using the following command: + +.. code:: bash + + export RELEASE="yy.mm" # e.g. export RELEASE="24.06" + + docker run -it --net=host --gpus=all nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk + + # Check out genai_perf command inside the container: + genai-perf --help + +.. raw:: html + +
+ +Alternatively, to install from source: + +Since GenAI-Perf depends on Perf Analyzer, you’ll need to install the +Perf Analyzer binary: + +Install Perf Analyzer (Ubuntu, Python 3.8+) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**NOTE**: you must already have CUDA 12 installed (checkout the `CUDA +installation +guide `__). + +.. code:: bash + + pip install tritonclient + + apt update && apt install -y --no-install-recommends libb64-0d libcurl4 + +You can also build Perf Analyzer `from +source <../docs/install.md#build-from-source>`__ as well. + +Install GenAI-Perf from source +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + git clone https://github.com/triton-inference-server/perf_analyzer.git && cd perf_analyzer + + pip install -e genai-perf + +.. raw:: html + +
+ +.. raw:: html + + + +Quick Start +----------- + +In this quick start, we will use GenAI-Perf to run performance +benchmarking on the GPT-2 model running on Triton Inference Server with +a TensorRT-LLM engine. + +Serve GPT-2 TensorRT-LLM model using Triton CLI +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can follow the `quickstart +guide `__ +on Triton CLI github repo to run GPT-2 model locally. The full +instructions are copied below for convenience: + +.. code:: bash + + # This container comes with all of the dependencies for building TRT-LLM engines + # and serving the engine with Triton Inference Server. + docker run -ti \ + --gpus all \ + --network=host \ + --shm-size=1g --ulimit memlock=-1 \ + -v /tmp:/tmp \ + -v ${HOME}/models:/root/models \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + nvcr.io/nvidia/tritonserver:24.05-trtllm-python-py3 + + # Install the Triton CLI + pip install git+https://github.com/triton-inference-server/triton_cli.git@0.0.8 + + # Build TRT LLM engine and generate a Triton model repository pointing at it + triton remove -m all + triton import -m gpt2 --backend tensorrtllm + + # Start Triton pointing at the default model repository + triton start + +Running GenAI-Perf +~~~~~~~~~~~~~~~~~~ + +Now we can run GenAI-Perf from Triton Inference Server SDK container: + +.. code:: bash + + export RELEASE="yy.mm" # e.g. export RELEASE="24.06" + + docker run -it --net=host --rm --gpus=all nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk + + # Run GenAI-Perf in the container: + genai-perf profile \ + -m gpt2 \ + --service-kind triton \ + --backend tensorrtllm \ + --num-prompts 100 \ + --random-seed 123 \ + --synthetic-input-tokens-mean 200 \ + --synthetic-input-tokens-stddev 0 \ + --streaming \ + --output-tokens-mean 100 \ + --output-tokens-stddev 0 \ + --output-tokens-mean-deterministic \ + --tokenizer hf-internal-testing/llama-tokenizer \ + --concurrency 1 \ + --measurement-interval 4000 \ + --profile-export-file my_profile_export.json \ + --url localhost:8001 + +Example output: + +:: + + LLM Metrics + ┏━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ + ┃ Statistic ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p75 ┃ + ┡━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ + │ Time to first token (ms) │ 11.70 │ 9.88 │ 17.21 │ 14.35 │ 12.01 │ 11.87 │ + │ Inter token latency (ms) │ 1.46 │ 1.08 │ 1.89 │ 1.87 │ 1.62 │ 1.52 │ + │ Request latency (ms) │ 161.24 │ 153.45 │ 200.74 │ 200.66 │ 179.43 │ 162.23 │ + │ Output sequence length │ 103.39 │ 95.00 │ 134.00 │ 120.08 │ 107.30 │ 105.00 │ + │ Input sequence length │ 200.01 │ 200.00 │ 201.00 │ 200.13 │ 200.00 │ 200.00 │ + └──────────────────────────┴────────┴────────┴────────┴────────┴────────┴────────┘ + Output token throughput (per sec): 635.61 + Request throughput (per sec): 6.15 + +See `Tutorial `__ for additional examples. + +.. raw:: html + + + +Visualization +------------- + +GenAI-Perf can also generate various plots that visualize the +performance of the current profile run. This is disabled by default but +users can easily enable it by passing the ``--generate-plots`` option +when running the benchmark: + +.. code:: bash + + genai-perf profile \ + -m gpt2 \ + --service-kind triton \ + --backend tensorrtllm \ + --streaming \ + --concurrency 1 \ + --generate-plots + +This will generate a `set of default +plots `__ such as: - Time to first token +(TTFT) analysis - Request latency analysis - TTFT vs Input sequence +lengths - Inter token latencies vs Token positions - Input sequence +lengths vs Output sequence lengths + +Using ``compare`` Subcommand to Visualize Multiple Runs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``compare`` subcommand in GenAI-Perf facilitates users in comparing +multiple profile runs and visualizing the differences through plots. + +Usage +^^^^^ + +Assuming the user possesses two profile export JSON files, namely +``profile1.json`` and ``profile2.json``, they can execute the +``compare`` subcommand using the ``--files`` option: + +.. code:: bash + + genai-perf compare --files profile1.json profile2.json + +Executing the above command will perform the following actions under the +``compare`` directory: 1. Generate a YAML configuration file +(e.g. ``config.yaml``) containing the metadata for each plot generated +during the comparison process. 2. Automatically generate the `default +set of plots `__ (e.g. TTFT vs. Input +Sequence Lengths) that compare the two profile runs. + +:: + + compare + ├── config.yaml + ├── distribution_of_input_sequence_lengths_to_output_sequence_lengths.jpeg + ├── request_latency.jpeg + ├── time_to_first_token.jpeg + ├── time_to_first_token_vs_input_sequence_lengths.jpeg + ├── token-to-token_latency_vs_output_token_position.jpeg + └── ... + +Customization +^^^^^^^^^^^^^ + +Users have the flexibility to iteratively modify the generated YAML +configuration file to suit their specific requirements. They can make +alterations to the plots according to their preferences and execute the +command with the ``--config`` option followed by the path to the +modified configuration file: + +.. code:: bash + + genai-perf compare --config compare/config.yaml + +This command will regenerate the plots based on the updated +configuration settings, enabling users to refine the visual +representation of the comparison results as per their needs. + +See `Compare documentation `__ for more details. + +.. raw:: html + + + +Model Inputs +------------ + +GenAI-Perf supports model input prompts from either synthetically +generated inputs, or from the HuggingFace +`OpenOrca `__ or +`CNN_DailyMail `__ +datasets. This is specified using the ``--input-dataset`` CLI option. + +When the dataset is synthetic, you can specify the following options: \* +``--num-prompts ``: The number of unique prompts to generate as +stimulus, >= 1. \* ``--synthetic-input-tokens-mean ``: The mean of +number of tokens in the generated prompts when using synthetic data, >= +1. \* ``--synthetic-input-tokens-stddev ``: The standard deviation +of number of tokens in the generated prompts when using synthetic data, +>= 0. \* ``--random-seed ``: The seed used to generate random +values, >= 0. + +When the dataset is coming from HuggingFace, you can specify the +following options: \* ``--input-dataset {openorca,cnn_dailymail}``: +HuggingFace dataset to use for benchmarking. \* ``--num-prompts ``: +The number of unique prompts to generate as stimulus, >= 1. + +When the dataset is coming from a file, you can specify the following +options: \* ``--input-file ``: The input file containing the +prompts to use for benchmarking as JSON objects. + +For any dataset, you can specify the following options: \* +``--output-tokens-mean ``: The mean number of tokens in each +output. Ensure the ``--tokenizer`` value is set correctly, >= 1. \* +``--output-tokens-stddev ``: The standard deviation of the number +of tokens in each output. This is only used when output-tokens-mean is +provided, >= 1. \* ``--output-tokens-mean-deterministic``: When using +``--output-tokens-mean``, this flag can be set to improve precision by +setting the minimum number of tokens equal to the requested number of +tokens. This is currently supported with the Triton service-kind. Note +that there is still some variability in the requested number of output +tokens, but GenAi-Perf attempts its best effort with your model to get +the right number of output tokens. + +You can optionally set additional model inputs with the following +option: \* ``--extra-inputs :``: An additional input +for use with the model with a singular value, such as ``stream:true`` or +``max_tokens:5``. This flag can be repeated to supply multiple extra +inputs. + +For `Large Language Models `__, there is no batch size +(i.e. batch size is always ``1``). Each request includes the inputs for +one individual inference. Other modes such as the +`embeddings `__ and `rankings `__ +endpoints support client-side batching, where ``--batch-size N`` means +that each request sent will include the inputs for ``N`` separate +inferences, allowing them to be processed together. + +.. raw:: html + + + +Metrics +------- + +GenAI-Perf collects a diverse set of metrics that captures the +performance of the inference server. + ++-----------------------+-----------------------+-----------------------+ +| Metric | Description | Aggregations | ++=======================+=======================+=======================+ +| Time to First Token | Time between when a | Avg, min, max, p99, | +| | request is sent and | p90, p75 | +| | when its first | | +| | response is received, | | +| | one value per request | | +| | in benchmark | | ++-----------------------+-----------------------+-----------------------+ +| Inter Token Latency | Time between | Avg, min, max, p99, | +| | intermediate | p90, p75 | +| | responses for a | | +| | single request | | +| | divided by the number | | +| | of generated tokens | | +| | of the latter | | +| | response, one value | | +| | per response per | | +| | request in benchmark | | ++-----------------------+-----------------------+-----------------------+ +| Request Latency | Time between when a | Avg, min, max, p99, | +| | request is sent and | p90, p75 | +| | when its final | | +| | response is received, | | +| | one value per request | | +| | in benchmark | | ++-----------------------+-----------------------+-----------------------+ +| Output Sequence | Total number of | Avg, min, max, p99, | +| Length | output tokens of a | p90, p75 | +| | request, one value | | +| | per request in | | +| | benchmark | | ++-----------------------+-----------------------+-----------------------+ +| Input Sequence Length | Total number of input | Avg, min, max, p99, | +| | tokens of a request, | p90, p75 | +| | one value per request | | +| | in benchmark | | ++-----------------------+-----------------------+-----------------------+ +| Output Token | Total number of | None–one value per | +| Throughput | output tokens from | benchmark | +| | benchmark divided by | | +| | benchmark duration | | ++-----------------------+-----------------------+-----------------------+ +| Request Throughput | Number of final | None–one value per | +| | responses from | benchmark | +| | benchmark divided by | | +| | benchmark duration | | ++-----------------------+-----------------------+-----------------------+ + +.. raw:: html + + + +Command Line Options +-------------------- + +``-h`` +'''''' + +``--help`` +'''''''''' + +Show the help message and exit. + +Endpoint Options: +~~~~~~~~~~~~~~~~~ + +``-m `` +''''''''''''' + +``--model `` +'''''''''''''''''' + +The names of the models to benchmark. A single model is recommended, +unless you are `profiling multiple LoRA adapters `__. +(default: ``None``) + +``--model-selection-strategy {round_robin, random}`` +'''''''''''''''''''''''''''''''''''''''''''''''''''' + +When multiple models are specified, this is how a specific model is +assigned to a prompt. Round robin means that each model receives a +request in order. Random means that assignment is uniformly random +(default: ``round_robin``) + +``--backend {tensorrtllm,vllm}`` +'''''''''''''''''''''''''''''''' + +When using the “triton” service-kind, this is the backend of the model. +For the TRT-LLM backend, you currently must set +``exclude_input_in_output`` to true in the model config to not echo the +input tokens in the output. (default: tensorrtllm) + +``--endpoint `` +'''''''''''''''''''' + +Set a custom endpoint that differs from the OpenAI defaults. (default: +``None``) + +``--endpoint-type {chat,completions,embeddings,rankings}`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The endpoint-type to send requests to on the server. This is only used +with the ``openai`` service-kind. (default: ``None``) + +``--service-kind {triton,openai}`` +'''''''''''''''''''''''''''''''''' + +The kind of service perf_analyzer will generate load for. In order to +use ``openai``, you must specify an api via ``--endpoint-type``. +(default: ``triton``) + +``--streaming`` +''''''''''''''' + +An option to enable the use of the streaming API. (default: ``False``) + +``-u `` +'''''''''''' + +``--url `` +''''''''''''''' + +URL of the endpoint to target for benchmarking. (default: ``None``) + +Input Options +~~~~~~~~~~~~~ + +``-b `` +'''''''''''' + +``--batch-size `` +'''''''''''''''''''''' + +The batch size of the requests GenAI-Perf should send. This is currently +only supported with the `embeddings `__, +image_retrieval, and `rankings `__ endpoint types. +(default: ``1``) + +``--extra-inputs `` +'''''''''''''''''''''''' + +Provide additional inputs to include with every request. You can repeat +this flag for multiple inputs. Inputs should be in an input_name:value +format. Alternatively, a string representing a json formatted dict can +be provided. (default: ``None``) + +``--input-dataset {openorca,cnn_dailymail}`` +'''''''''''''''''''''''''''''''''''''''''''' + +The HuggingFace dataset to use for prompts. (default: ``openorca``) + +``--input-file `` +''''''''''''''''''''''' + +The input file containing the prompts to use for profiling. Each line +should be a JSON object with a ‘text_input’ field in JSONL format. +Example: {"text_input": "Your prompt here"}" + +``--num-prompts `` +''''''''''''''''''''''' + +The number of unique prompts to generate as stimulus. (default: ``100``) + +``--output-tokens-mean `` +'''''''''''''''''''''''''''''' + +The mean number of tokens in each output. Ensure the ``--tokenizer`` +value is set correctly. (default: ``-1``) + +``--output-tokens-mean-deterministic`` +'''''''''''''''''''''''''''''''''''''' + +When using ``--output-tokens-mean``, this flag can be set to improve +precision by setting the minimum number of tokens equal to the requested +number of tokens. This is currently supported with the Triton +service-kind. Note that there is still some variability in the requested +number of output tokens, but GenAi-Perf attempts its best effort with +your model to get the right number of output tokens. (default: +``False``) + +``--output-tokens-stddev `` +'''''''''''''''''''''''''''''''' + +The standard deviation of the number of tokens in each output. This is +only used when ``--output-tokens-mean`` is provided. (default: ``0``) + +``--random-seed `` +''''''''''''''''''''''' + +The seed used to generate random values. (default: ``0``) + +``--synthetic-input-tokens-mean `` +''''''''''''''''''''''''''''''''''''''' + +The mean of number of tokens in the generated prompts when using +synthetic data. (default: ``550``) + +``--synthetic-input-tokens-stddev `` +''''''''''''''''''''''''''''''''''''''''' + +The standard deviation of number of tokens in the generated prompts when +using synthetic data. (default: ``0``) + +Profiling Options +~~~~~~~~~~~~~~~~~ + +``--concurrency `` +''''''''''''''''''''''' + +The concurrency value to benchmark. (default: ``None``) + +``--measurement-interval `` +'''''''''''''''''''''''''''''''' + +``-p `` +'''''''''''' + +The time interval used for each measurement in milliseconds. Perf +Analyzer will sample a time interval specified and take measurement over +the requests completed within that time interval. (default: ``10000``) + +``--request-rate `` +'''''''''''''''''''''''''' + +Sets the request rate for the load generated by PA. (default: ``None``) + +``-s `` +'''''''''''''' + +``--stability-percentage `` +'''''''''''''''''''''''''''''''''' + +The allowed variation in latency measurements when determining if a +result is stable. The measurement is considered as stable if the ratio +of max / min from the recent 3 measurements is within (stability +percentage) in terms of both infer per second and latency. (default: +``999``) + +Output Options +~~~~~~~~~~~~~~ + +``--artifact-dir`` +'''''''''''''''''' + +The directory to store all the (output) artifacts generated by +GenAI-Perf and Perf Analyzer. (default: ``artifacts``) + +``--generate-plots`` +'''''''''''''''''''' + +An option to enable the generation of plots. (default: False) + +``--profile-export-file `` +'''''''''''''''''''''''''''''''' + +The path where the perf_analyzer profile export will be generated. By +default, the profile export will be to ``profile_export.json``. The +genai-perf files will be exported to +``_genai_perf.json`` and +``_genai_perf.csv``. For example, if the profile +export file is ``profile_export.json``, the genai-perf file will be +exported to ``profile_export_genai_perf.csv``. (default: +``profile_export.json``) + +Other Options +~~~~~~~~~~~~~ + +``--tokenizer `` +''''''''''''''''''''' + +The HuggingFace tokenizer to use to interpret token metrics from prompts +and responses. (default: ``hf-internal-testing/llama-tokenizer``) + +``-v`` +'''''' + +``--verbose`` +''''''''''''' + +An option to enable verbose mode. (default: ``False``) + +``--version`` +''''''''''''' + +An option to print the version and exit. + +.. raw:: html + + + +Known Issues +------------ + +- GenAI-Perf can be slow to finish if a high request-rate is provided +- Token counts may not be exact diff --git a/docs/perf_benchmark/genai_perf.rst b/docs/perf_benchmark/genai_perf.rst index 65de2d8ce1..175662477f 100644 --- a/docs/perf_benchmark/genai_perf.rst +++ b/docs/perf_benchmark/genai_perf.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -28,12 +28,13 @@ #### GenAI Performance Analyzer #### +.. include:: genai-perf-README.rst + .. toctree:: :maxdepth: 1 :hidden: - Overview <../perf_analyzer/genai-perf/README.md> Large language models <../perf_analyzer/genai-perf/docs/tutorial.md> Visual language models <../perf_analyzer/genai-perf/docs/multi_modal.md> Embedding models <../perf_analyzer/genai-perf/docs/embeddings.md> diff --git a/docs/perf_benchmark/model-analyzer-README.rst b/docs/perf_benchmark/model-analyzer-README.rst new file mode 100644 index 0000000000..f31e7ca633 --- /dev/null +++ b/docs/perf_benchmark/model-analyzer-README.rst @@ -0,0 +1,203 @@ +.. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +|License| + +Triton Model Analyzer +===================== + + [!Warning] + + .. rubric:: LATEST RELEASE + :name: latest-release + + You are currently on the ``main`` branch which tracks + under-development progress towards the next release. The latest + release of the Triton Model Analyzer is 1.42.0 and is available on + branch + `r24.07 `__. + +Triton Model Analyzer is a CLI tool which can help you find a more +optimal configuration, on a given piece of hardware, for single, +multiple, ensemble, or BLS models running on a `Triton Inference +Server `__. Model +Analyzer will also generate reports to help you better understand the +trade-offs of the different configurations along with their compute and +memory requirements. + +Features +======== + +Search Modes +~~~~~~~~~~~~ + +- `Optuna Search `__ **-ALPHA + RELEASE-** allows you to search for every parameter that can be + specified in the model configuration, using a hyperparameter + optimization framework. Please see the + `Optuna `__ website if you are interested in + specific details on how the algorithm functions. + +- `Quick Search `__ will + **sparsely** search the `Max Batch + Size `__, + `Dynamic + Batching `__, + and `Instance + Group `__ + spaces by utilizing a heuristic hill-climbing algorithm to help you + quickly find a more optimal configuration + +- `Automatic Brute + Search `__ will + **exhaustively** search the `Max Batch + Size `__, + `Dynamic + Batching `__, + and `Instance + Group `__ + parameters of your model configuration + +- `Manual Brute Search `__ + allows you to create manual sweeps for every parameter that can be + specified in the model configuration + +Model Types +~~~~~~~~~~~ + +- `Ensemble `__: Model Analyzer can help + you find the optimal settings when profiling an ensemble model + +- `BLS `__: Model Analyzer can help you find + the optimal settings when profiling a BLS model + +- `Multi-Model `__: Model Analyzer can + help you find the optimal settings when profiling multiple concurrent + models + +- `LLM `__: Model Analyzer can help you find + the optimal settings when profiling Large Language Models + +Other Features +~~~~~~~~~~~~~~ + +- `Detailed and summary reports `__: Model Analyzer is + able to generate summarized and detailed reports that can help you + better understand the trade-offs between different model + configurations that can be used for your model. + +- `QoS Constraints `__: Constraints can help + you filter out the Model Analyzer results based on your QoS + requirements. For example, you can specify a latency budget to filter + out model configurations that do not satisfy the specified latency + threshold. + +Examples and Tutorials +====================== + +**Single Model** +~~~~~~~~~~~~~~~~ + +See the `Single Model Quick Start `__ for a guide +on how to use Model Analyzer to profile, analyze and report on a simple +PyTorch model. + +**Multi Model** +~~~~~~~~~~~~~~~ + +See the `Multi-model Quick Start `__ for a guide +on how to use Model Analyzer to profile, analyze and report on two +models running concurrently on the same GPU. + +**Ensemble Model** +~~~~~~~~~~~~~~~~~~ + +See the `Ensemble Model Quick Start `__ +for a guide on how to use Model Analyzer to profile, analyze and report +on a simple Ensemble model. + +**BLS Model** +~~~~~~~~~~~~~ + +See the `BLS Model Quick Start `__ for a guide +on how to use Model Analyzer to profile, analyze and report on a simple +BLS model. + +Documentation +============= + +- `Installation `__ +- `Model Analyzer CLI `__ +- `Launch Modes `__ +- `Configuring Model Analyzer `__ +- `Model Analyzer Metrics `__ +- `Model Config Search `__ +- `Model Types `__ +- `Checkpointing `__ +- `Model Analyzer Reports `__ +- `Deployment with Kubernetes `__ + +Terminology +=========== + +Below are definitions of some commonly used terms in Model Analyzer: + +- **Model Type** - Category of model being profiled. Examples of this + include single, multi, ensemble, BLS, etc.. +- **Search Mode** - How Model Analyzer explores the possible + configuration space when profiling. This is either exhaustive (brute) + or heuristic (quick/optuna). +- **Model Config Search** - The cross product of model type and search + mode. +- **Launch Mode** - How the Triton Server is deployed and used by Model + Analyzer. + +Reporting problems, asking questions +==================================== + +We appreciate any feedback, questions or bug reporting regarding this +project. When help with code is needed, follow the process outlined in +the Stack Overflow (https://stackoverflow.com/help/mcve) document. +Ensure posted examples are: + +- minimal – use as little code as possible that still produces the same + problem + +- complete – provide all parts needed to reproduce the problem. Check + if you can strip external dependency and still show the problem. The + less time we spend on reproducing problems the more time we have to + fix it + +- verifiable – test the code you’re about to provide to make sure it + reproduces the problem. Remove all other problems that are not + related to your request/question. + +.. |License| image:: https://img.shields.io/badge/License-Apache_2.0-lightgrey.svg + :target: https://opensource.org/licenses/Apache-2.0 diff --git a/docs/perf_benchmark/model_analyzer.rst b/docs/perf_benchmark/model_analyzer.rst index a3c734ce30..c29a96aa92 100644 --- a/docs/perf_benchmark/model_analyzer.rst +++ b/docs/perf_benchmark/model_analyzer.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -29,23 +29,17 @@ Model Analyzer #### +.. include:: model-analyzer-README.rst + .. toctree:: :maxdepth: 1 :hidden: - Overview <../model_analyzer/README.md> - Documentation <../model_analyzer/docs/README.md> - Quick Start <../model_analyzer/docs/quick_start.md> - Installation <../model_analyzer/docs/install.md> - CLI Reference <../model_analyzer/docs/cli.md> - Launch Modes <../model_analyzer/docs/launch_modes.md> - Configuration <../model_analyzer/docs/config.md> - Configuration Search <../model_analyzer/docs/config_search.md> - Metrics <../model_analyzer/docs/metrics.md> - Checkpointing <../model_analyzer/docs/checkpoints.md> - Reports <../model_analyzer/docs/report.md> - Kubernetes <../model_analyzer/docs/kubernetes_deploy.md> - Model Types <../model_analyzer/docs/model_types.md> - Ensemble Model <../model_analyzer/docs/ensemble_quick_start.md> - BLS Model <../model_analyzer/docs/bls_quick_start.md> - Multi-Model <../model_analyzer/docs/mm_quick_start.md> \ No newline at end of file + ../model_analyzer/docs/cli.md + ../model_analyzer/docs/launch_modes.md + ../model_analyzer/docs/config.md + ../model_analyzer/docs/metrics.md + ../model_analyzer/docs/config_search.md + ../model_analyzer/docs/checkpoints.md + ../model_analyzer/docs/report.md + ../model_analyzer/docs/kubernetes_deploy.md \ No newline at end of file diff --git a/docs/perf_benchmark/perf-analyzer-README.rst b/docs/perf_benchmark/perf-analyzer-README.rst new file mode 100644 index 0000000000..4f678cfdba --- /dev/null +++ b/docs/perf_benchmark/perf-analyzer-README.rst @@ -0,0 +1,180 @@ +.. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +Triton Performance Analyzer +=========================== + +Triton Performance Analyzer is CLI tool which can help you optimize the +inference performance of models running on Triton Inference Server by +measuring changes in performance as you experiment with different +optimization strategies. + +Features +======== + +Inference Load Modes +~~~~~~~~~~~~~~~~~~~~ + +- `Concurrency Mode `__ + simlulates load by maintaining a specific concurrency of outgoing + requests to the server + +- `Request Rate + Mode `__ simulates + load by sending consecutive requests at a specific rate to the server + +- `Custom Interval + Mode `__ simulates + load by sending consecutive requests at specific intervals to the + server + +Performance Measurement Modes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- `Time Windows Mode `__ + measures model performance repeatedly over a specific time interval + until performance has stabilized + +- `Count Windows Mode `__ + measures model performance repeatedly over a specific number of + requests until performance has stabilized + +Other Features +~~~~~~~~~~~~~~ + +- `Sequence Models <../user_guide/architecture.md#stateful-models>`__, + `Ensemble Models <../user_guide/architecture.md#ensemble-models>`__, + and `Decoupled Models <../user_guide/decoupled_models.md>`__ can be + profiled in addition to standard/stateless/coupled models + +- `Input Data `__ to model inferences can be + auto-generated or specified as well as verifying output + +- `TensorFlow + Serving `__ and + `TorchServe `__ can be + used as the inference server in addition to the default Triton server + +Quick Start +=========== + +The steps below will guide you on how to start using Perf Analyzer. + +Step 1: Start Triton Container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + export RELEASE= # e.g. to use the release from the end of February of 2023, do `export RELEASE=23.02` + + docker pull nvcr.io/nvidia/tritonserver:${RELEASE}-py3 + + docker run --gpus all --rm -it --net host nvcr.io/nvidia/tritonserver:${RELEASE}-py3 + +Step 2: Download ``simple`` Model +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + # inside triton container + git clone --depth 1 https://github.com/triton-inference-server/server + + mkdir model_repository ; cp -r server/docs/examples/model_repository/simple model_repository + +Step 3: Start Triton Server +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + # inside triton container + tritonserver --model-repository $(pwd)/model_repository &> server.log & + + # confirm server is ready, look for 'HTTP/1.1 200 OK' + curl -v localhost:8000/v2/health/ready + + # detach (CTRL-p CTRL-q) + +Step 4: Start Triton SDK Container +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + docker pull nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk + + docker run --gpus all --rm -it --net host nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk + +Step 5: Run Perf Analyzer +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + # inside sdk container + perf_analyzer -m simple + +See the full `quick start guide `__ for additional +tips on how to analyze output. + +Documentation +============= + +- `Installation `__ +- `Perf Analyzer CLI `__ +- `Inference Load Modes `__ +- `Input Data `__ +- `Measurements & Metrics `__ +- `Benchmarking `__ + +Contributing +============ + +Contributions to Triton Perf Analyzer are more than welcome. To +contribute please review the `contribution +guidelines `__, +then fork and create a pull request. + +Reporting problems, asking questions +==================================== + +We appreciate any feedback, questions or bug reporting regarding this +project. When help with code is needed, follow the process outlined in +the Stack Overflow (https://stackoverflow.com/help/mcve) document. +Ensure posted examples are: + +- minimal - use as little code as possible that still produces the same + problem + +- complete - provide all parts needed to reproduce the problem. Check + if you can strip external dependency and still show the problem. The + less time we spend on reproducing problems the more time we have to + fix it + +- verifiable - test the code you’re about to provide to make sure it + reproduces the problem. Remove all other problems that are not + related to your request/question. diff --git a/docs/perf_benchmark/perf_analyzer.rst b/docs/perf_benchmark/perf_analyzer.rst index fcb25e0ca8..d6c6156a62 100644 --- a/docs/perf_benchmark/perf_analyzer.rst +++ b/docs/perf_benchmark/perf_analyzer.rst @@ -1,5 +1,5 @@ .. -.. Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. .. .. Redistribution and use in source and binary forms, with or without .. modification, are permitted provided that the following conditions @@ -29,16 +29,14 @@ Performance Analyzer #### +.. include:: perf-analyzer-README.rst + .. toctree:: :maxdepth: 1 :hidden: - Overview <../perf_analyzer/README.md> - Documentation <../perf_analyzer/docs/README.md> - Quick Start <../perf_analyzer/docs/quick_start.md> - Installation <../perf_analyzer/docs/install.md> - CLI Reference <../perf_analyzer/docs/cli.md> - Inference Load Modes <../perf_analyzer/docs/inference_load_modes.md> - Input Data <../perf_analyzer/docs/input_data.md> - Measurement Modes <../perf_analyzer/docs/measurements_metrics.md> - Benchmarking <../perf_analyzer/docs/benchmarking.md> \ No newline at end of file + ../perf_analyzer/docs/install.md + ../perf_analyzer/docs/CLI.md + ../perf_analyzer/docs/inference_load_modes.md + ../perf_analyzer/docs/input_data.md + ../perf_analyzer/docs/measurements_metrics.md \ No newline at end of file diff --git a/docs/protocol/extension_parameters.md b/docs/protocol/extension_parameters.md index 17edcf2776..14ed4d1dc5 100644 --- a/docs/protocol/extension_parameters.md +++ b/docs/protocol/extension_parameters.md @@ -1,5 +1,5 @@ -# \ - -**Status**: \[Draft | Under Review | Approved | Replaced | Deferred | Rejected\] - -**Authors**: \[Name/Team\] - -**Category**: \[Architecture | Process | Guidelines\] - -**Replaces**: \[Link of previous proposal if applicable\] - -**Replaced By**: \[Link of previous proposal if applicable\] - -**Sponsor**: \[Name of code owner or maintainer to shepherd process\] - -**Required Reviewers**: \[Names of technical leads that are required for acceptance\] - -**Review Date**: \[Date for review\] - -**Pull Request**: \[Link to Pull Request of the Proposal itself\] - -**Implementation PR / Tracking Issue**: \[Link to Pull Request or Tracking Issue for Implementation\] - -## Summary - -**\[Required\]** - -## Motivation - -**\[Required\]** - -Describe the problem that needs to be addressed with enough detail for someone familiar with the project to understand. -Generally one to two short paragraphs. -Additional details can be placed in the background section as needed. -Cover **what** the issue is and **why** it needs to be addressed. -Link to github issues if relevant. - -### Goals - -**\[Optional \- if not applicable omit\]** - -List out any additional goals in bullet points. -Goals may be aspirational / difficult to measure but guide the proposal. - -* Goal - -* Goal - -* Goal - -### Non Goals - -**\[Optional \- if not applicable omit\]** - -List out any items which are out of scope / specifically not required in bullet points. -Indicates the scope of the proposal and issue being resolved. - -### Requirements - -**\[Optional \- if not applicable omit\]** - -List out any additional requirements in numbered subheadings. - -**\** - -#### REQ \<\#\> \ - -Describe the requirement in as much detail as necessary for others to understand it and how it applies to the TEP. -Keep in mind that requirements should be measurable and will be used to determine if a TEP has been successfully implemented or not. - -Requirement names should be prefixed using a monotonically increasing number such as “REQ 1 \” followed by “REQ 2 \” and so on. -Use title casing when naming requirements. Requirement names should be as descriptive as possible while remaining as terse as possible. - -Use all-caps, bolded terms like **MUST** and **SHOULD** when describing each requirement. -See \[RFC-2119\](https://datatracker.ietf.org/doc/html/rfc2119) for additional information. - -## Proposal - -**\[Required\]** - -Describe the high level design / proposal. -Use sub sections as needed, but start with an overview and then dig into the details. -Try to provide images and diagrams to facilitate understanding. - -## Implementation Details - -**\[Optional \- if not applicable omit\]** - -Add additional detailed items here including interface signatures, etc. -Add anything that is relevant but seems more of a detail than central to the proposal. -Use sub sections / bullet points as needed. -Try to provide images and diagrams to facilitate understanding. -If applicable link to PR. - -### Deferred to Implementation - -**\[Optional \- if not applicable omit\]** - -List out items that are under discussion but that will be resolved only during implementation / code review. - -## Implementation Phases - -**\[Optional \- if not applicable omit\]** - -List out phases of implementation (can be single phase). -Give each phase a monotonically increasing number; example “Phase 0” followed by “Phase 1” and so on. -Give phases titles if it makes sense. - -### Phase \<\#\> \ - -**Release Target**: Date - -**Effort Estimate**: \ - -**Work Item(s):** \ - -**Supported API / Behavior:** - -* \ - -**Not Supported:** - -* \ - -## Related Proposals - -**\[Optional \- if not applicable omit\]** - -* File - -* File - -* File - -* File - -* File - -## Alternate Solutions - -**\[Required, if not applicable write N/A\]** - -List out solutions that were considered but ultimately rejected. -Consider free form `-`, but a possible format shown below. - -### Alt \<\#\> \ - -**Pros:** - -\ - -**Cons:** - -\ - -**Reason Rejected:** - -\ - -**Notes:** - -\ - -## Background - -**\[Optional \- if not applicable omit\]** - -Add additional context and references as needed to help reviewers and authors understand the context of the problem and solution being proposed. - -## References - -**\[Optional \- if not applicable omit\]** - -Add additional references as needed to help reviewers and authors understand the context of the problem and solution being proposed. - -* \ - -## Terminology & Definitions - -**\[Optional \- if not applicable omit\]** - -List out additional terms / definitions (lexicon). -Try to keep definitions as concise as possible and use links to external resources when additional information would be useful to the reader. - -Keep the list of terms sorted alphabetically to ease looking up definitions by readers. - -| \ | \ | -| :---- | :---- | -| **\** | \ | - -## Acronyms & Abbreviations - -**\[Optional \- if not applicable omit\]** - -Provide a list of frequently used acronyms and abbreviations which are uncommon or unlikely to be known by the reader. -Do not include acronyms or abbreviations which the reader is likely to be familiar with. - -Keep the list of acronyms and abbreviations sorted alphabetically to ease looking up definitions by readers. - -Do not include the full definition in the expanded meaning of an abbreviation or acronym. -If the reader needs the definition, please include it in the \[Terminology & Definitions\](#terminology--definitions) section. - -**\:** \ diff --git a/enhancements/NNNN-template-limited.md b/enhancements/NNNN-template-limited.md deleted file mode 100644 index f814dce593..0000000000 --- a/enhancements/NNNN-template-limited.md +++ /dev/null @@ -1,134 +0,0 @@ - -# \ - -**Status**: \[Draft | Under Review | Approved | Replaced | Deferred | Rejected\] - -**Authors**: \[Name/Team\] - -**Category**: \[Architecture | Process | Guidelines\] - -**Replaces**: \[Link of previous proposal if applicable\] - -**Replaced By**: \[Link of previous proposal if applicable\] - -**Sponsor**: \[Name of code owner or maintainer to shepherd process\] - -**Required Reviewers**: \[Names of technical leads that are required for acceptance\] - -**Review Date**: \[Date for review\] - -**Pull Request**: \[Link to Pull Request of the Proposal itself\] - -**Implementation PR / Tracking Issue**: \[Link to Pull Request or Tracking Issue for Implementation\] - -## Summary - -**\[Required\]** - -## Motivation - -**\[Required\]** - -Describe the problem that needs to be addressed with enough detail for someone familiar with the project to understand. -Generally one to two short paragraphs. -Additional details can be placed in the background section as needed. Cover **what** the issue is and **why** it needs to be addressed. -Link to github issues if relevant. - -### Goals - -**\[Optional \- if not applicable omit\]** - -List out any additional goals in bullet points. -Goals may be aspirational / difficult to measure but guide the proposal. - -* Goal - -* Goal - -* Goal - -#### Non Goals - -**\[Optional \- if not applicable omit\]** - -List out any items which are out of scope / specifically not required in bullet points. -Indicates the scope of the proposal and issue being resolved. - -### Requirements - -**\[Optional \- if not applicable omit\]** - -List out any additional requirements in numbered subheadings. - -**\** - -#### REQ \<\#\> \ - -Describe the requirement in as much detail as necessary for others to understand it and how it applies to the TEP. -Keep in mind that requirements should be measurable and will be used to determine if a TEP has been successfully implemented or not. - -Requirement names should be prefixed using a monotonically increasing number such as “REQ 1 \” followed by “REQ 2 \” and so on. -Use title casing when naming requirements. -Requirement names should be as descriptive as possible while remaining as terse as possible. - -Use all-caps, bolded terms like **MUST** and **SHOULD** when describing each requirement. -See \[RFC-2119\](https://datatracker.ietf.org/doc/html/rfc2119) for additional information. - -## Proposal - -**\[Required\]** - -Describe the high level design / proposal. -Use sub sections as needed, but start with an overview and then dig into the details. -Try to provide images and diagrams to facilitate understanding. - -## Alternate Solutions - -**\[Required, if not applicable write N/A\]** - -List out solutions that were considered but ultimately rejected. -Consider free form `-`, but a possible format shown below. - -## Alt \<\#\> \ - -**Pros:** - -\ - -**Cons:** - -\ - -**Reason Rejected:** - -\ - -**Notes:** - -\ diff --git a/enhancements/README.md b/enhancements/README.md deleted file mode 100644 index 6cf8c3a1da..0000000000 --- a/enhancements/README.md +++ /dev/null @@ -1,44 +0,0 @@ - -# Triton Enhancement Proposals (TEP) - -Enhancement Proposals and Architecture Decisions - -Please see [0000-tep-process](teps/0000-tep-process.md) for full explanation and details. - -## Authoring Guidelines - -1. Start with either the: -- [NNNN-template-complete.md](NNNN-template-complete.md) and remove unneeded sections. -- [NNNN-template-limited.md](NNNN-template-limited.md) and then add selectively from the complete template based on need. - -1. Identify a **Code-Owner** or **Maintainer** of the TEP repository to shepherd the process. - -2. Create a draft PR and iterate with co-authors, **Sponsor** - -3. When ready for review, mark as ready and work with **Sponsor** to set a **Review Date**. diff --git a/enhancements/teps/0000-tep-process.md b/enhancements/teps/0000-tep-process.md deleted file mode 100644 index b85c538dcf..0000000000 --- a/enhancements/teps/0000-tep-process.md +++ /dev/null @@ -1,265 +0,0 @@ - -# Triton Enhancement Proposals - -**Status**: Draft - -**Authors**: [whoisj](https://github.com/whoisj) - -**Category**: Process - -**Replaces**: N/A - -**Replaced By**: N/A - -**Sponsor**: [whoisj](https://github.com/whoisj) - -**Required Reviewers**: [dzier](https://github.com/dzier), [nnshah1](https://github.com/nnshah1) - -**Review Date**: 17 Oct 2025 - -**Pull Request**: [N/A](https://github.com/triton-inference-server/server/pull/8517) - -## Summary - -A standard process and format for proposing and capturing architecture, design, and process decisions for the Triton project along with the motivations behind those decisions. -We adopt a similar process as adopted by Dynamo, Kubernetes, Rust, Python, and Ray broadly categorized as "enhancement proposals". - -## Motivation - -With any software project but especially agile, open source projects in the AI space, architecture, design, and process decisions are made rapidly and for specific reasons which can sometimes be difficult to understand after the fact. -For Triton in particular many teams and community members are collaborating for the first time and have varied backgrounds and design philosophies. -The Triton project's code base itself reflects multiple previously independent code bases integrated quickly to meet overall project goals. -As the project evolves we need a way to propose, ratify and capture architecture, design and process decisions quickly and thoughtfully in a transparent, consistent, lightweight, maintainable way. - -Borrowing from the motivation for KEPs: - -> The purpose of the KEP process is to reduce the amount of "tribal knowledge" in our community. -> By moving decisions from a smattering of mailing lists, video calls and hallway conversations into a well tracked artifact, this process aims to enhance communication and discoverability. - -### Goals - -* **Useful** - - Enhancement proposals and the process of writing and approving them should encourage the thoughtful evaluation of design, process, and architecture choices and lead to timely decisions with a clear record of what was decided, why, and what other options were considered. - -* **Lightweight and Scalable** - - The format and process should be applicable both to small or medium sized changes as well as large ones. - The process should not impede the rate of progress but serve to provide timely feedback, discussion, and ratification on key proposals. - The process should also support retroactive documents to capture and explain decisions already made. - -* **Single Document for Requirements and Design** - - Combine aspects of requirements documents, design documents and software architecture documents into a single document. - Give one place to understand the motivation, requirements, and design of a feature or process. - -* **Support Process, Architecture and Guideline Decisions** - - Have a single format to articulate decisions that effect process (such as github merge rules or templates) as well as code and design guidelines as well as features. - -* **Clear** - - Should be relatively clear when a document is required, when the review needs to be completed, and by who and what the overall process is. - -* **Encourage Collaboration** - - Should allow for easy collaboration and communication between *Authors** and **Reviewers**. - -* **Flexible** - - Format and process should be flexible enough to be used for different types of decisions requiring different levels of detail and formatting of sections. - -### Non Goals - -* Triton Enhancement Proposals (TEP)s do not take the place of other forms of documentation such as user / developer facing documentation (including architecture documents, api documentation) -* Prototyping and early development are not gated by design / architectural approval. -* TEPs should not be a perfunctory process but lead to discussion and thought process around good designs. -* Not all changes (bug fixes, documentation improvements) need a TEP - and many can be reviewed via that normal GitHub pull request - -## Proposal - -Following successful open source projects such as Kubernetes (KEP) and Dynamo (DEP) we adopt a markdown based enhancement proposal format designed to support any decisions we need to capture as a project. -We will adopt an open, community-wide, discussion and comment process using pull requests but enable **Code-Owners** and **Maintainers** to be the final arbiters of **Approval**. - -Subject area experts will be listed as required **Reviewers** to ensure proposals are complete and reviewed properly. - - - -We provide two templates "limited" and "complete" where the limited template is a strict subset of the complete template, and both indicate which sections are required and which are optional. - -## Implementation Details - -### Proposal Process - - - -* Copy the [limited template](../NNNN-template-limited.md) or [complete template](../NNNN-template-complete.md) to `teps/NNNN-my-feature.md` (where `my-feature` is descriptive, don't assign an `TEP` identifier yet) - - > [!Note] - > Choose the template that fits your purpose. - > You can start with the limited form and pull additional sections from the complete form as needed. - > Keep the order of the sections consistent. - -* Identify a **Sponsor** from the list of **Maintainers** or **Code-Owners** to help with the process. - -* Fill in the proposal template. - Be sure to include all required sections. - Keep sections in the order prescribed in the template. - -* Work with the **Sponsor** to identify the required reviewers and a timeline for review. - - - -* If discussion is needed the **Sponsor** can ask for a slot in the weekly Engineering Sync or schedule an ad-hoc meeting with the required reviewers. - -* Iterate and incorporate feedback via the pull request. - -* When review is complete The **Sponsor** will merge the request and update the status. - -* **Sponsor** should assign an identifier. - -* **Author** and **Sponsor** should add issues and/or PRs as needed to track implementation. - -### When is a proposal required? - -It is difficult to enumerate all the circumstances where a proposal would be required or not required. -Generally we will follow this process when making "substantial changes". -The definition of "substantial" is evolving and mainly determined by the core team and community. - -When in doubt reach out to a **Maintainer** or **Code-Owner**. - -**Generally speaking a proposal would not be required for**: - -* Bug fixes that don't change advertised behavior - -* Documentation fixes / updates - -* Minor refactors within a single module - -**Generally speaking proposals would be required for**: - -* New features which add significant functionality - -* Changes to existing features or code which require discussion - -* Changes to public interfaces - -* Responses to security related vulnerabilities found directly in the project code - -* Changes to packaging and installation - -* When a **Maintainer** or **Code-Owner** recommends that a change go through the proposal process - -* Retroactively to capture current architecture, guideline, or process - -### Minor Changes After Review - -For minor changes or changes that are in the spirit of the review, updates can be made to the document without a new proposal. - -*Example:* links to implementation - -### Significant Changes After Review - -For significant changes, a new proposal should be made and the original marked as replaced. - -### Maintenance - -TEPs should be reviewed for updates, replacements, or archiving on a regular basis. - -### Sensitive Changes and Discussions - -Certain types of changes need to be discussed and ratified before being made public due to timing of non-disclosed information. -In such (rare) cases, drafts and reviews will be conducted offline by **Authors**, **Code-Owners**, and **Maintainers** with the public proposals being updated when possible. - -*Example:* when responding to undisclosed security vulnerabilities, we want to avoid inadvertently encouraging zero day attacks for deployed systems. - -In such (rare) cases, we may make use of a private repo on a temporary basis to collect feedback before publishing to the public repo. - -### Deferred to Implementation - -* Definition of **Code-Owners** and **Maintainers** - -* Whether or not to organize **TEP**s into sub directories for projects / areas - -* Tooling around the creation / indexing of **TEP**s - -* Making requirements required in addition to motivation - -* Format recommendations for API surfaces / other formatted components. - -* Decisions / guidelines on when a TEP is needed. - -## Alternate Solutions - -### Alt 1 Google Docs - -**Pros:** - -* Fits existing documents and templates used by many teams - -**Cons:** - -* Difficult to integrate with AI tools. - -* Difficult to search and index - -**Reason Rejected:** - -* Want to standardize around a simple text format and use AI tools also for diagramming, etc. - -## Background - -With the rise of Agile software development practices and large open source projects, software development teams needed to devise new and lightweight (w.r.t to previous software architecture documents) ways of recording architecture proposals and decisions. -As Agile was born in part as a reaction to waterfall styles of planning and development and famously prioritized “Working software over comprehensive documentation”, so too there was a need to replace monolithic large software design specifications with something lighter weight but that still encouraged good architecture. - -From this need for a new way of practicing software architecture a body of work and theory has evolved around the concepts of “Architecture Decision Records” which in turn are also termed “Any Decision Record”, and RFCs or Enhancement proposals (PEP, KEP, REP). - -In each case the core requirements of the process are that the team document the problem, the proposal / design, the status of the proposal, implications / follow on work, and any alternatives that were considered using a standard template and review process. - -Just as in Agile planning, each team modifies the template and process to fit their needs. - -### References - -1. [Documenting Architecture Decisions (cognitect.com)](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions) - -2. [The most plagiarized Architecture Decision Record blog on the internet. | by Conall Daly | Medium](https://conalldalydev.medium.com/the-most-plagiarised-architecture-decision-record-blog-on-the-internet-c9dd2018c1d6) - -3. [adr.github.io](https://adr.github.io/) - -4. [When Should I Write an Architecture Decision Record \- Spotify Engineering : Spotify Engineering (atspotify.com)](https://engineering.atspotify.com/2020/04/when-should-i-write-an-architecture-decision-record/) - -5. [Scaling Engineering Teams via RFCs: Writing Things Down \- The Pragmatic Engineer](https://blog.pragmaticengineer.com/scaling-engineering-teams-via-writing-things-down-rfcs/) - -6. [Love Unrequited: The Story of Architecture, Agile, and How Architecture Decision Records Brought Them Together | IEEE Journals & Magazine | IEEE Xplore](https://ieeexplore.ieee.org/document/9801811) - -7. [ray-project/enhancements: Tracking Ray Enhancement Proposals (github.com)](https://github.com/ray-project/enhancements) - -8. [Kubernetes Enhancement Proposals](https://github.com/kubernetes/enhancements/blob/master/keps/sig-architecture/0000-kep-process/README.md) - -9. [Dynamo Enhancement Proposals](https://github.com/ai-dynamo/enhancements/blob/main/README.md) diff --git a/python/openai/README.md b/python/openai/README.md index 92d91d4db7..53a9f461f4 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -1,5 +1,5 @@ -# OpenAI-Compatible Frontend for Triton Inference Server +# OpenAI-Compatible Frontend for Triton Inference Server (Beta) + +> [!NOTE] +> The OpenAI-Compatible API is currently in BETA. Its features and functionality +> are subject to change as we collect feedback. We're excited to hear any thoughts +> you have and what features you'd like to see! ## Pre-requisites @@ -38,7 +43,7 @@ ## VLLM 1. Launch the container and install dependencies: - - Mounts the `~/.cache/huggingface` for re-use of downloaded models across runs, containers, etc. + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to access gated models, make sure this is set in your local environment if needed. @@ -46,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -93,7 +98,7 @@ curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/ ```json { - "id": "cmpl-0242093d-51ae-11f0-b339-e7480668bfbe", + "id": "cmpl-6930b296-7ef8-11ef-bdd1-107c6149ca79", "choices": [ { "finish_reason": "stop", @@ -108,15 +113,11 @@ curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/ "logprobs": null } ], - "created": 1750846825, + "created": 1727679085, "model": "llama-3.1-8b-instruct", "system_fingerprint": null, "object": "chat.completion", - "usage": { - "completion_tokens": 7, - "prompt_tokens": 42, - "total_tokens": 49 - } + "usage": null } ``` @@ -137,32 +138,28 @@ curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' ```json { - "id": "cmpl-58fba3a0-51ae-11f0-859d-e7480668bfbe", + "id": "cmpl-d51df75c-7ef8-11ef-bdd1-107c6149ca79", "choices": [ { "finish_reason": "stop", "index": 0, "logprobs": null, - "text": " an amazing field that can truly understand the hidden patterns that exist in the data," + "text": " a field of computer science that focuses on developing algorithms that allow computers to learn from" } ], - "created": 1750846970, + "created": 1727679266, "model": "llama-3.1-8b-instruct", "system_fingerprint": null, "object": "text_completion", - "usage": { - "completion_tokens": 16, - "prompt_tokens": 4, - "total_tokens": 20 - } + "usage": null } ```
5. Benchmark with `genai-perf`: -- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-genai-perf-ubuntu-2404-python-310) -- Or try using genai-perf from the [SDK container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tritonserver) +- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) +- Or try using genai-perf from the [SDK container](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) ```bash MODEL="llama-3.1-8b-instruct" @@ -219,7 +216,7 @@ completion = client.chat.completions.create( }, {"role": "user", "content": "What are LLMs?"}, ], - max_completion_tokens=256, + max_tokens=256, ) print(completion.choices[0].message.content) @@ -233,208 +230,6 @@ pip install -r requirements-test.txt pytest -v tests/ ``` -### LoRA Adapters - -If the command line argument `--lora-separator=` is provided -when starting the OpenAI Frontend, a LoRA adaptor listed in `multi_lora.json` -may be selected by appending the LoRA name to the model name, -separated by the LoRA separator, on the inference request in -`` format. - -
-For example - -```bash -# start server with model named gemma-2b -python3 openai_frontend/main.py --lora-separator=_lora_ ... - -# inference without LoRA -curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ - "model": "gemma-2b", - "temperature": 0, - "prompt": "When was the wheel invented?" -}' -{ - ... - "choices":[{..."text":"\n\nThe wheel was invented by the Sumerians in Mesopotamia around 350"}], - ... -} - -# inference with LoRA named doll -curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ - "model": "gemma-2b_lora_doll", - "temperature": 0, - "prompt": "When was the wheel invented?" -}' -{ - ... - "choices":[{..."text":"\n\nThe wheel was invented in Mesopotamia around 3500 BC.\n\n"}], - ... -} - -# inference with LoRA named sheep -curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ - "model": "gemma-2b_lora_sheep", - "temperature": 0, - "prompt": "When was the wheel invented?" -}' -{ - ... - "choices":[{..."text":"\n\nThe wheel was invented around 3000 BC in Mesopotamia.\n\n"}], - ... -} -``` - -
- -When listing or retrieving model(s), the model id will include the LoRA name in -the same `` format for each LoRA -adapter listed on the `multi_lora.json`. Note: The LoRA name inclusion is -limited to locally stored models, inference requests are not limited though. - -#### vLLM -See the -[vLLM documentation](https://github.com/triton-inference-server/vllm_backend/blob/main/docs/llama_multi_lora_tutorial.md) -on how to serve a vLLM model with LoRA adapters. - -#### TensorRT-LLM -Similarly, see [TensorRT-LLM document](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/docs/lora.md) -on how to prepare LoRA-enabled TensorRT-LLM engines and generate LoRA tensors. -The path of LoRA adapter in `multi_lora.json` is the directory of -`model.lora_config.npy` and `model.lora_weights.npy` tensors. - -
-For example - -model repository -``` -inflight_batcher_llm -├── postprocessing -| ├── 1 -| | └── model.py -| └── config.pbtxt -├── preprocessing -| ├── 1 -| | └── model.py -| └── config.pbtxt -├── tensorrt_llm -| ├── 1 -| | └── model.py -| └── config.pbtxt -└── tensorrt_llm_bls - ├── 1 - | ├── Japanese-Alpaca-LoRA-7b-v0-weights - | | ├── model.lora_config.npy - | | └── model.lora_weights.npy - | ├── luotuo-lora-7b-0.1-weights - | | ├── model.lora_config.npy - | | └── model.lora_weights.npy - | ├── model.py - | └── multi_lora.json - └── config.pbtxt -``` - -multi_lora.json -``` -{ - "doll": "inflight_batcher_llm/tensorrt_llm_bls/1/luotuo-lora-7b-0.1-weights", - "sheep": "inflight_batcher_llm/tensorrt_llm_bls/1/Japanese-Alpaca-LoRA-7b-v0-weights" -} -``` -
- -### Embedding Models -Currently, OpenAI-Compatible Frontend supports loading embedding models and embeddings endpoints via vLLM backend. Check [vLLM supported models](https://docs.vllm.ai/en/latest/models/supported_models.html#embedding) for all supported embedding models from vLLM. - -1. Launch the container and install dependencies: - - Mounts the `~/.cache/huggingface` for re-use of downloaded models across runs, containers, etc. - - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to - access gated models, make sure this is set in your local environment if needed. - -```bash -docker run -it --net=host --gpus all --rm \ - -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ - -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 -``` - -2. Launch the OpenAI-compatible Triton Inference Server: -```bash -cd /opt/tritonserver/python/openai - -# NOTE: Embeddings endpoint does not require "--tokenizer" -python3 openai_frontend/main.py --model-repository tests/vllm_embedding_models -``` - -
-Example output - -``` -... -+------------------+---------+--------+ -| Model | Version | Status | -+------------------+---------+--------+ -| all-MiniLM-L6-v2 | 1 | READY | <- Correct Model Loaded in Triton -+------------------+---------+--------+ -... -Found model: name='all-MiniLM-L6-v2', backend='vllm' -[WARNING] Adding CORS for the following origins: ['http://localhost'] -INFO: Started server process [133] -INFO: Waiting for application startup. -INFO: Application startup complete. -INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully -``` - -
- -3. Send a `/v1/embeddings` request: - - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. -```bash -MODEL="all-MiniLM-L6-v2" -curl -s http://localhost:9000/v1/embeddings \ - -H 'Content-Type: application/json' \ - -d '{ - "model": "'${MODEL}'", - "input": "The food was delicious and the waiter...", - "dimensions": 10, - "encoding_format": "float" - }' | jq -``` - -
-Example output - -```json -{ - "object": "list", - "data": [ - { - "object": "embedding", - "embedding": [ - -0.1914404183626175, - 0.4000193178653717, - 0.058502197265625, - 0.18909454345703125, - -0.4690297544002533, - 0.004936536308377981, - 0.45893096923828125, - -0.31141534447669983, - 0.18299102783203125, - -0.4907582700252533 - ], - "index": 0 - } - ], - "model": "all-MiniLM-L6-v2", - "usage": { - "prompt_tokens": 12, - "total_tokens": 12 - } -} -``` - -
- ## TensorRT-LLM 0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: @@ -442,7 +237,7 @@ curl -s http://localhost:9000/v1/embeddings \ - [TRT-LLM Backend Quickstart](https://github.com/triton-inference-server/tensorrtllm_backend?tab=readme-ov-file#quick-start) 1. Launch the container: - - Mounts the `~/.cache/huggingface` for re-use of downloaded models across runs, containers, etc. + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to access gated models, make sure this is set in your local environment if needed. @@ -451,7 +246,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ -e TRTLLM_ORCHESTRATOR=1 \ - nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 + nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 ``` 2. Install dependencies inside the container: @@ -488,13 +283,13 @@ curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/ ```json { - "id": "cmpl-5ad4f860-bf13-11f0-b137-b75b7f0a8586", + "id": "cmpl-704c758c-8a84-11ef-b106-107c6149ca79", "choices": [ { "finish_reason": "stop", "index": 0, "message": { - "content": "It looks like you're ready to see if I'm functioning properly. What would", + "content": "It looks like you're testing the system!", "tool_calls": null, "role": "assistant", "function_call": null @@ -502,15 +297,11 @@ curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/ "logprobs": null } ], - "created": 1762875029, - "model": "tensorrt_llm_bls", + "created": 1728948689, + "model": "llama-3-8b-instruct", "system_fingerprint": null, "object": "chat.completion", - "usage": { - "prompt_tokens": 42, - "total_tokens": 58, - "completion_tokens": 16 - } + "usage": null } ``` @@ -542,125 +333,6 @@ available arguments and default values. For more information on the `tritonfrontend` python bindings, see the docs [here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md). -## Model Management - -The OpenAI-compatible frontend supports explicit model control, allowing you to -dynamically load and unload models at runtime without restarting the server. -This is particularly useful when hosting multiple large models on a shared GPU -cluster and needing to swap models on demand. - -### Model Control Mode - -Use `--model-control-mode` to specify how models are managed at startup and -runtime. The default is `none`. - -| Mode | Behavior | -|------|----------| -| `none` (default) | All models in the repository are loaded at startup. Load/unload APIs are not available. | -| `explicit` | No models are loaded at startup unless specified with `--load-model`. Load and unload are controlled via the management API. | - -> [!NOTE] -> This matches the native `tritonserver --model-control-mode` behavior. -> See [Triton Model Management](../../docs/user_guide/model_management.md) for -> more details. - -### Load Models at Startup (Explicit Mode) - -When using `--model-control-mode=explicit`, use `--load-model` to specify which -models should be loaded at startup. It may be specified multiple times to load -multiple models. - -
-Example - -```bash -# Start in explicit mode with no models loaded -python3 openai_frontend/main.py \ - --model-repository /path/to/models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --model-control-mode explicit - -# Start in explicit mode and load a specific model at startup -python3 openai_frontend/main.py \ - --model-repository /path/to/models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --model-control-mode explicit \ - --load-model llama-3.1-8b-instruct - -# Load multiple models at startup -python3 openai_frontend/main.py \ - --model-repository /path/to/models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --model-control-mode explicit \ - --load-model model-a \ - --load-model model-b - -# Load ALL models in the repository at startup -python3 openai_frontend/main.py \ - --model-repository /path/to/models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --model-control-mode explicit \ - --load-model '*' -``` - -
- -> [!IMPORTANT] -> - `--load-model` requires `--model-control-mode=explicit`. -> - `--load-model=*` can not be used together with loading specific models. - -### Dynamic Load / Unload API - -Once the server is running in `explicit` mode, models can be loaded and unloaded -at runtime via the following endpoints: - -| Method | Endpoint | Description | -|--------|----------|-------------| -| `POST` | `/v1/models/{model_name}/load` | Load a model. Blocks until model is fully loaded and ready. | -| `POST` | `/v1/models/{model_name}/unload` | Unload a model. Blocks until fully unloaded. In-flight requests complete before removal. | - -Both endpoints return an error if `--model-control-mode` is not `explicit`. - -#### Load a model - -```bash -MODEL="llama-3.1-8b-instruct" -curl -s -X POST http://localhost:9000/v1/models/${MODEL}/load | jq -``` - -
-Example output - -```json -{ - "id": "llama-3.1-8b-instruct", - "object": "model", - "created": 1750000000, - "owned_by": "Triton Inference Server" -} -``` - -
- -#### Unload a model - -```bash -MODEL="llama-3.1-8b-instruct" -curl -s -X POST http://localhost:9000/v1/models/${MODEL}/unload | jq -``` - -
-Example output - -```json -{ - "status": "success", - "model": "llama-3.1-8b-instruct" -} -``` - -
- ## Model Parallelism Support - [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend)) @@ -670,307 +342,3 @@ curl -s -X POST http://localhost:9000/v1/models/${MODEL}/unload | jq - Set the following environment variable: `export TRTLLM_ORCHESTRATOR=1` - [ ] TensorRT-LLM ([Leader Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#leader-mode)) - Not currently supported - -## Tool Calling - -The OpenAI frontend supports `tools` and `tool_choice` in the `v1/chat/completions` API. Please refer to the OpenAI API reference for more details about these parameters: - [tools](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools), - [tool_choice](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice) - -To enable the tool-calling feature, add the `--tool-call-parser {parser_name}` flag when starting the server. The two available parsers are `llama3` and `mistral`. -The `llama3` parser supports tool-calling features for LLaMA 3.1, 3.2, and 3.3 models, while the `mistral` parser supports tool-calling features for the Mistral Instruct model. - -Example for launching the OpenAI frontend with a tool call parser: -``` -python3 openai_frontend/main.py \ - --model-repository tests/vllm_models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --tool-call-parser llama3 -``` - -Example for making a tool calling request: - -```python -import json -from openai import OpenAI - - -def get_current_weather(city: str, state: str, unit: "str"): - return ( - "The weather in Dallas, Texas is 85 degrees fahrenheit. It is " - "partly cloudly, with highs in the 90's." - ) - -available_tools = {"get_current_weather": get_current_weather} - -openai_api_key = "EMPTY" -openai_api_base = "http://localhost:9000/v1" - -client = OpenAI( - api_key=openai_api_key, - base_url=openai_api_base, -) - -model = "llama-3.1-8b-instruct" # change this to the model in the repository - -tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to find the weather for, e.g. 'San Francisco'", - }, - "state": { - "type": "string", - "description": "the two-letter abbreviation for the state that the city is" - " in, e.g. 'CA' which would mean 'California'", - }, - "unit": { - "type": "string", - "description": "The unit to fetch the temperature in", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["city", "state", "unit"], - }, - }, - } -] - -messages = [ - { - "role": "system", - "content": "You're a helpful assistant! Answer the users question best you can.", - }, - {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"}, -] - -tool_calls = client.chat.completions.create( - messages=messages, model=model, tools=tools, max_completion_tokens=128 -) -function_name = tool_calls.choices[0].message.tool_calls[0].function.name -function_arguments = tool_calls.choices[0].message.tool_calls[0].function.arguments - -print(f"function name: " f"{function_name}") -print(f"function arguments: {function_arguments}") -print(f"tool calling result: {available_tools[function_name](**json.loads(function_arguments))}") -``` - -Example output: -``` -function name: get_current_weather -function arguments: {"city": "Dallas", "state": "TX", "unit": "fahrenheit"} -tool calling result: The weather in Dallas, Texas is 85 degrees fahrenheit. It is partly cloudly, with highs in the 90's. -``` - -#### Named Tool Calling - -The OpenAI frontend supports named function calling, utilizing structured outputs in the vLLM backend and guided decoding in TensorRT-LLM backend. Users can specify one of the tools in `tool_choice` to force the model to select a specific tool for function calling. - -> [!NOTE] -> For instructions on enabling guided decoding in the TensorRT-LLM backend, please refer to [this guide](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/docs/guided_decoding.md) - -Example for making a named tool calling request: - -```python -import json -from openai import OpenAI - - -def get_current_weather(city: str, state: str, unit: "str"): - return ( - "The weather in Dallas, Texas is 85 degrees fahrenheit. It is " - "partly cloudly, with highs in the 90's." - ) - -def get_n_day_weather_forecast(city: str, state: str, unit: str, num_days: int): - return ( - f"The weather in Dallas, Texas is 85 degrees fahrenheit in next {num_days} days." - ) - -available_tools = {"get_current_weather": get_current_weather, - "get_n_day_weather_forecast": get_n_day_weather_forecast} - -openai_api_key = "EMPTY" -openai_api_base = "http://localhost:9000/v1" -client = OpenAI( - api_key=openai_api_key, - base_url=openai_api_base, -) -model = "llama-3.1-8b-instruct" # change this to the model in the repository -tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to find the weather for, e.g. 'San Francisco'", - }, - "state": { - "type": "string", - "description": "the two-letter abbreviation for the state that the city is" - " in, e.g. 'CA' which would mean 'California'", - }, - "unit": { - "type": "string", - "description": "The unit to fetch the temperature in", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["city", "state", "unit"], - }, - }, - }, - { - "type": "function", - "function": { - "name": "get_n_day_weather_forecast", - "description": "Get an N-day weather forecast", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to find the weather for, " - "e.g. 'San Francisco'", - }, - "state": { - "type": "string", - "description": "must the two-letter abbreviation for the state " - "that the city is in, e.g. 'CA' which would " - "mean 'California'", - }, - "unit": { - "type": "string", - "description": "The unit to fetch the temperature in", - "enum": ["celsius", "fahrenheit"], - }, - "num_days": { - "type": "integer", - "description": "The number of days to forecast", - }, - }, - "required": ["city", "state", "unit", "num_days"], - }, - }, - } -] - -tool_choice = {"function": {"name": "get_n_day_weather_forecast"}, "type": "function"} - -messages = [ - { - "role": "system", - "content": "You're a helpful assistant! Answer the users question best you can.", - }, - {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"}, -] - -tool_calls = client.chat.completions.create( - messages=messages, model=model, tools=tools, tool_choice=tool_choice, max_completion_tokens=128 -) -function_name = tool_calls.choices[0].message.tool_calls[0].function.name -function_arguments = tool_calls.choices[0].message.tool_calls[0].function.arguments - -print(f"function name: {function_name}") -print(f"function arguments: {function_arguments}") -print(f"tool calling result: {available_tools[function_name](**json.loads(function_arguments))}") -``` - -Example output: -``` -function name: get_n_day_weather_forecast -function arguments: {"city": "Dallas", "state": "TX", "unit": "fahrenheit", num_days: 1} -tool calling result: The weather in Dallas, Texas is 85 degrees fahrenheit in next 1 days. -``` - -## Limit Endpoint Access - -The OpenAI-compatible server supports restricting access to specific API endpoints through authentication headers. This feature allows you to protect sensitive endpoints while keeping others publicly accessible. - -### Configuration - -Use the `--openai-restricted-api` command-line argument to configure endpoint restrictions: - -``` ---openai-restricted-api ,,... -``` - -- **`API`**: A comma-separated list of APIs to be included in this group. Note that currently a given API is not allowed to be included in multiple groups. The following protocols / APIs are recognized: - - **inference**: Chat completions and text completions endpoints - - `POST /v1/chat/completions` - - `POST /v1/completions` - - **embedding**: Embedding endpoint - - `POST /v1/embeddings` - - **model-repository**: Model listing, information, and dynamic load/unload endpoints - - `GET /v1/models` - - `GET /v1/models/{model_name}` - - `POST /v1/models/{model_name}/load` - - `POST /v1/models/{model_name}/unload` - - **metrics**: Server metrics endpoint - - `GET /metrics` - - **health**: Health check endpoint - - `GET /health/ready` - -- **`restricted-key`**: The HTTP request header to be checked when a request is received. -- **`restricted-value`**: The header value required to access the specified protocols. - -### Examples - -#### Restrict Inference API Endpoints Only -```bash ---openai-restricted-api "inference api-key my-secret-key" -``` - -Clients must include the header: -```bash -curl -H "api-key: my-secret-key" \ - -X POST http://localhost:9000/v1/chat/completions \ - -d '{"model": "my-model", "messages": [{"role": "user", "content": "Hello"}]}' -``` - -#### Restrict Multiple API Endpoints -```bash -# Different authentication for different APIs ---openai-restricted-api "inference user-key user-secret" \ ---openai-restricted-api "model-repository admin-key admin-secret" - -# Multiple APIs in single argument with shared authentication ---openai-restricted-api "inference,model-repository shared-key shared-secret" -``` - -## HTTP Request Body Size Limit - -The frontend enforces a maximum request body size prior to JSON parsing. Requests that exceed this limit are rejected with an error response. - -Use `--http-max-input-size` to configure the limit (default: `67108864` bytes / 64 MiB): - -```bash -python3 openai_frontend/main.py \ - --model-repository /path/to/models \ - --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ - --http-max-input-size 67108864 -``` - -The limit applies to all endpoints. Example error response: - -```json -{ - "error": { - "message": "Request content size exceeds the maximum allowed input size of 67108864 bytes. Use --http-max-input-size to increase the limit.", - "type": "invalid_request_error", - "code": "content_too_large" - } -} -``` diff --git a/python/openai/openai_frontend/engine/engine.py b/python/openai/openai_frontend/engine/engine.py index da68ac024b..9c90dec25e 100644 --- a/python/openai/openai_frontend/engine/engine.py +++ b/python/openai/openai_frontend/engine/engine.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,8 +34,6 @@ CreateChatCompletionResponse, CreateCompletionRequest, CreateCompletionResponse, - CreateEmbeddingRequest, - CreateEmbeddingResponse, Model, ) @@ -94,25 +92,3 @@ def completion( If request.stream is False, this returns a CreateCompletionResponse. """ pass - - def embedding(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: - """ - Returns a CreateEmbeddingResponse. - """ - pass - - async def load_model(self, model_name: str) -> Model: - """ - Loads a model by name. Only available in EXPLICIT model control mode. - Blocks until the model is fully loaded and ready, matching standard - Triton server load behavior. - """ - pass - - async def unload_model(self, model_name: str) -> None: - """ - Unloads a model by name. Only available in EXPLICIT model control mode. - Blocks until the model is fully unloaded, matching standard Triton - server unload behavior. In-flight requests complete before unload. - """ - pass diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 19a5b2f157..e315cbbfd8 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,74 +27,36 @@ from __future__ import annotations -import asyncio -import base64 -import json import time import uuid from dataclasses import dataclass -from typing import ( - Any, - AsyncIterable, - AsyncIterator, - Callable, - Dict, - List, - Literal, - Optional, - Tuple, - Union, -) +from typing import Any, AsyncIterable, AsyncIterator, Callable, Dict, List, Optional -import numpy as np import tritonserver from engine.engine import LLMEngine -from engine.utils.chat import load_chat_template, parse_chat_messages from engine.utils.tokenizer import get_tokenizer -from engine.utils.tool_call_parsers import ToolCallParser, ToolParserManager from engine.utils.triton import ( - RequestKind, - TritonLoraConfig, - _create_trtllm_embedding_request, - _create_trtllm_generate_request, - _create_vllm_embedding_request, - _create_vllm_generate_request, - _get_openai_chat_format_logprobs_from_vllm_response, - _get_openai_completion_format_logprobs_from_vllm_response, + _create_trtllm_inference_request, + _create_vllm_inference_request, _get_output, - _get_usage_from_response, - _parse_lora_configs, - _StreamingUsageAccumulator, _validate_triton_responses_non_streaming, ) from schemas.openai import ( ChatCompletionChoice, ChatCompletionFinishReason, - ChatCompletionLogprobs, - ChatCompletionMessageToolCall, - ChatCompletionMessageToolCallChunk, - ChatCompletionNamedToolChoice, ChatCompletionResponseMessage, ChatCompletionStreamingResponseChoice, ChatCompletionStreamResponseDelta, - ChatCompletionToolChoiceOption1, Choice, - CompletionUsage, CreateChatCompletionRequest, CreateChatCompletionResponse, CreateChatCompletionStreamResponse, CreateCompletionRequest, CreateCompletionResponse, - CreateEmbeddingRequest, - CreateEmbeddingResponse, - EmbeddingObject, FinishReason, - Function1, - Function2, Model, ObjectType, ) -from utils.utils import ClientError, ServerError # TODO: Improve type hints @@ -108,44 +70,26 @@ class TritonModelMetadata: model: tritonserver.Model # Tokenizers used for chat templates tokenizer: Optional[Any] - # LoRA names supported by the backend - lora_configs: Optional[List[TritonLoraConfig]] - # Name of the input tensor enabling "echo" parameter in /v1/completions endpoint - echo_tensor_name: Optional[str] # Time that model was loaded by Triton create_time: int # Conversion format between OpenAI and Triton requests - inference_request_converter: Callable - embedding_request_converter: Callable + request_converter: Callable class TritonLLMEngine(LLMEngine): def __init__( - self, - server: tritonserver.Server, - tokenizer: str, - default_max_tokens: int, - backend: Optional[str] = None, - lora_separator: Optional[str] = None, - tool_call_parser: Optional[str] = None, - chat_template: Optional[str] = None, + self, server: tritonserver.Server, tokenizer: str, backend: Optional[str] = None ): # Assume an already configured and started server self.server = server self.tokenizer = self._get_tokenizer(tokenizer) # TODO: Reconsider name of "backend" vs. something like "request_format" self.backend = backend - self.lora_separator = lora_separator - self.default_max_tokens = default_max_tokens + # NOTE: Creation time and model metadata will be static at startup for + # now, and won't account for dynamically loading/unloading models. + self.create_time = int(time.time()) self.model_metadata = self._get_model_metadata() - self._metadata_lock = asyncio.Lock() - self.tool_call_parser = ( - ToolParserManager.get_tool_parser_cls(tool_call_parser) - if tool_call_parser - else None - ) - self.chat_template = load_chat_template(chat_template) def ready(self) -> bool: return self.server.ready() @@ -156,64 +100,37 @@ def metrics(self) -> str: def models(self) -> List[Model]: models = [] for metadata in self.model_metadata.values(): - model_names = [metadata.name] - if ( - self.lora_separator is not None - and len(self.lora_separator) > 0 - and metadata.lora_configs is not None - ): - for lora_config in metadata.lora_configs: - model_names.append( - f"{metadata.name}{self.lora_separator}{lora_config.name}" - ) - - for model_name in model_names: - models.append( - Model( - id=model_name, - created=metadata.create_time, - object=ObjectType.model, - owned_by="Triton Inference Server", - ), - ) + models.append( + Model( + id=metadata.name, + created=metadata.create_time, + object=ObjectType.model, + owned_by="Triton Inference Server", + ), + ) return models async def chat( self, request: CreateChatCompletionRequest ) -> CreateChatCompletionResponse | AsyncIterator[str]: - model_name, lora_name = self._get_model_and_lora_name(request.model) - metadata = self.model_metadata.get(model_name) - self._validate_chat_request(request, metadata, lora_name) - - conversation = parse_chat_messages(request.messages) + metadata = self.model_metadata.get(request.model) + self._validate_chat_request(request, metadata) + conversation = [ + message.model_dump(exclude_none=True) for message in request.messages + ] add_generation_prompt = True - tool_dicts = ( - None - if request.tools is None - else [tool.model_dump() for tool in request.tools] - ) - prompt = metadata.tokenizer.apply_chat_template( conversation=conversation, tokenize=False, add_generation_prompt=add_generation_prompt, - tools=tool_dicts, - chat_template=self.chat_template, ) # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.inference_request_converter( - metadata.model, - prompt, - request, - self._get_lora_config(model_name, lora_name), - metadata.echo_tensor_name, - self.default_max_tokens, - ) + metadata.request_converter(metadata.model, prompt, request) ) # Prepare and send responses back to client in OpenAI format @@ -224,19 +141,9 @@ async def chat( conversation, add_generation_prompt, default_role ) - tool_call_parser = ( - self.tool_call_parser(metadata.tokenizer) if self.tool_call_parser else None - ) - if request.stream: return self._streaming_chat_iterator( - request_id, - metadata.backend, - created, - request, - role, - tool_call_parser, - responses, + request_id, created, request.model, role, responses ) # Response validation with decoupled models in mind @@ -245,113 +152,34 @@ async def chat( response = responses[0] text = _get_output(response) - response_message, finish_reason = self._get_chat_completion_response_message( - request=request, - request_id=request_id, - tool_call_parser=tool_call_parser, - text=text, - role=role, - backend=metadata.backend, - ) - - usage = _get_usage_from_response( - response, metadata.backend, RequestKind.GENERATION - ) - - # Parse logprobs if requested - logprobs_data = None - if request.logprobs: - openai_logprobs = _get_openai_chat_format_logprobs_from_vllm_response( - response - ) - if openai_logprobs: - logprobs_data = ChatCompletionLogprobs(content=openai_logprobs) - return CreateChatCompletionResponse( id=request_id, choices=[ ChatCompletionChoice( index=0, - message=response_message, - logprobs=logprobs_data, - finish_reason=finish_reason, + message=ChatCompletionResponseMessage( + content=text, role=role, function_call=None + ), + logprobs=None, + finish_reason=ChatCompletionFinishReason.stop, ) ], created=created, model=request.model, system_fingerprint=None, object=ObjectType.chat_completion, - usage=usage, - ) - - def _get_chat_completion_response_message( - self, - request: CreateChatCompletionRequest, - request_id: str, - tool_call_parser: ToolCallParser, - text: str, - role: str, - backend: str, - ) -> Tuple[ChatCompletionResponseMessage, ChatCompletionFinishReason]: - response_message: ChatCompletionResponseMessage - auto_tools_called = False - tool_function_name = self._get_named_function_name(request=request) - if tool_function_name: - response_message = ChatCompletionResponseMessage( - content="", - role=role, - tool_calls=[ - ChatCompletionMessageToolCall( - id=request_id, - type="function", - function=Function1(name=tool_function_name, arguments=text), - ) - ], - ) - elif ( - tool_call_parser - and request.tools - and ( - request.tool_choice is None - or request.tool_choice.root == ChatCompletionToolChoiceOption1.auto - ) - ): - response_message = tool_call_parser.parse_tool_calls(text, role, backend) - auto_tools_called = ( - response_message.tool_calls is not None - and len(response_message.tool_calls.root) > 0 - ) - else: - response_message = ChatCompletionResponseMessage( - content=text, role=role, tool_calls=None - ) - - finish_reason = ( - ChatCompletionFinishReason.tool_calls - if auto_tools_called - else ChatCompletionFinishReason.stop ) - return response_message, finish_reason - async def completion( self, request: CreateCompletionRequest ) -> CreateCompletionResponse | AsyncIterator[str]: # Validate request and convert to Triton format - model_name, lora_name = self._get_model_and_lora_name(request.model) - metadata = self.model_metadata.get(model_name) - self._validate_completion_request(request, metadata, lora_name) + metadata = self.model_metadata.get(request.model) + self._validate_completion_request(request, metadata) # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.inference_request_converter( - metadata.model, - request.prompt, - request, - self._get_lora_config(model_name, lora_name), - metadata.echo_tensor_name, - self.default_max_tokens, - ) + metadata.request_converter(metadata.model, request.prompt, request) ) # Prepare and send responses back to client in OpenAI format @@ -359,7 +187,7 @@ async def completion( created = int(time.time()) if request.stream: return self._streaming_completion_iterator( - request_id, created, request, responses, metadata.backend + request_id, created, metadata.name, responses ) # Response validation with decoupled models in mind @@ -368,21 +196,10 @@ async def completion( response = responses[0] text = _get_output(response) - usage = _get_usage_from_response( - response, metadata.backend, RequestKind.GENERATION - ) - - # Parse logprobs if requested - logprobs_data = None - if request.logprobs is not None and request.logprobs > 0: - logprobs_data = _get_openai_completion_format_logprobs_from_vllm_response( - response - ) - choice = Choice( finish_reason=FinishReason.stop, index=0, - logprobs=logprobs_data, + logprobs=None, text=text, ) return CreateCompletionResponse( @@ -391,61 +208,9 @@ async def completion( system_fingerprint=None, object=ObjectType.text_completion, created=created, - model=request.model, - usage=usage, + model=metadata.name, ) - async def embedding( - self, request: CreateEmbeddingRequest - ) -> CreateEmbeddingResponse: - # Validate request and convert to Triton format - model_name, _ = self._get_model_and_lora_name(request.model) - metadata = self.model_metadata.get(model_name) - self._validate_embedding_request(request, metadata) - - # Convert to Triton request format and perform inference - responses = metadata.model.async_infer( - metadata.embedding_request_converter( - metadata.model, - request, - ) - ) - - # Response validation with decoupled models in mind - responses = [response async for response in responses] - _validate_triton_responses_non_streaming(responses) - response = responses[0] - - # Extract embedding from response (currently stored as JSON string in text_output) - embedding_json = _get_output(response) - embedding_list = json.loads(embedding_json) - - usage = _get_usage_from_response( - response, metadata.backend, RequestKind.EMBEDDING - ) - - embedding = self._get_embedding(embedding_list, request.encoding_format) - embedding_obj = EmbeddingObject( - embedding=embedding, index=0, object="embedding" - ) - - return CreateEmbeddingResponse( - object="list", - data=[embedding_obj], - model=request.model, - usage=usage, - ) - - @staticmethod - def _get_embedding( - embedding: List[float], encoding_format: Literal["float", "base64"] - ) -> Union[list[float], str]: - if encoding_format == "float": - return embedding - else: - embedding_bytes = np.array(embedding, dtype="float32").tobytes() - return base64.b64encode(embedding_bytes).decode("utf-8") - # TODO: This behavior should be tested further def _get_first_response_role( self, conversation: List[Dict], add_generation_prompt: bool, default_role: str @@ -456,34 +221,18 @@ def _get_first_response_role( return conversation[-1]["role"] # TODO: Expose explicit flag to catch edge cases - def _determine_request_converter(self, backend: str, request_type: RequestKind): + def _determine_request_converter(self, backend: str): # Allow manual override of backend request format if provided by user if self.backend: backend = self.backend # Request conversion from OpenAI format to backend-specific format if backend == "vllm": - if request_type == RequestKind.GENERATION: - return _create_vllm_generate_request - else: - return _create_vllm_embedding_request + return _create_vllm_inference_request # Use TRT-LLM format as default for everything else. This could be # an ensemble, a python or BLS model, a TRT-LLM backend model, etc. - if request_type == RequestKind.GENERATION: - return _create_trtllm_generate_request - else: - return _create_trtllm_embedding_request - - def _get_model_and_lora_name(self, request_model_name: str): - if self.lora_separator is None or len(self.lora_separator) == 0: - return request_model_name, None - - names = request_model_name.split(self.lora_separator) - if len(names) != 2: - return request_model_name, None - - return names[0], names[1] + return _create_trtllm_inference_request def _get_tokenizer(self, tokenizer_name: str): tokenizer = None @@ -492,114 +241,30 @@ def _get_tokenizer(self, tokenizer_name: str): return tokenizer - def _build_model_metadata(self, name: str) -> TritonModelMetadata: - model = self.server.model(name) - backend = model.config()["backend"] - if not backend and model.config()["platform"] == "ensemble": - backend = "ensemble" - print(f"Found model: {name=}, {backend=}") - - lora_configs = _parse_lora_configs( - self.server.options.model_repository, - name, - model.version, - backend if self.backend is None else self.backend, - ) - - echo_tensor_name = None - for input in model.config()["input"]: - if input["name"] in [ - "exclude_input_in_output", - "sampling_param_exclude_input_from_output", - ]: - echo_tensor_name = input["name"] - break - - return TritonModelMetadata( - name=name, - backend=backend, - model=model, - tokenizer=self.tokenizer, - lora_configs=lora_configs, - echo_tensor_name=echo_tensor_name, - create_time=int(time.time()), - inference_request_converter=self._determine_request_converter( - backend, RequestKind.GENERATION - ), - embedding_request_converter=self._determine_request_converter( - backend, RequestKind.EMBEDDING - ), - ) - def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]: - # One tokenizer is shared for all loaded models; creation time is per model. + # One tokenizer and creation time shared for all loaded models for now. model_metadata = {} - for name, _ in self.server.models(exclude_not_ready=True).keys(): - model_metadata[name] = self._build_model_metadata(name) - return model_metadata - - async def load_model(self, model_name: str) -> Model: - if ( - self.server.options.model_control_mode - != tritonserver.ModelControlMode.EXPLICIT - ): - raise ClientError( - "Model load/unload requires --model-control-mode=explicit" - ) - - async with self._metadata_lock: - if model_name in self.model_metadata: - raise ClientError(f"Model '{model_name}' is already loaded") - - # Blocking C API call dispatched to thread pool to avoid blocking - # the event loop. The C API blocks until model is fully loaded and - # ready, matching standard Triton server behavior. - try: - metadata = await asyncio.to_thread(self._load_model_sync, model_name) - except tritonserver.InvalidArgumentError as e: - raise ClientError(f"Failed to load model '{model_name}': {e}") - except tritonserver.TritonError as e: - raise ServerError(f"Failed to load model '{model_name}': {e}") - - self.model_metadata[model_name] = metadata - return Model( - id=model_name, - created=metadata.create_time, - object=ObjectType.model, - owned_by="Triton Inference Server", - ) - - def _load_model_sync(self, model_name: str) -> TritonModelMetadata: - self.server.load(model_name) - return self._build_model_metadata(model_name) - - async def unload_model(self, model_name: str) -> None: - if ( - self.server.options.model_control_mode - != tritonserver.ModelControlMode.EXPLICIT - ): - raise ClientError( - "Model load/unload requires --model-control-mode=explicit" + # Read all triton models and store the necessary metadata for each + for name, _ in self.server.models().keys(): + model = self.server.model(name) + backend = model.config()["backend"] + # Explicitly handle ensembles to avoid any runtime validation errors + if not backend and model.config()["platform"] == "ensemble": + backend = "ensemble" + print(f"Found model: {name=}, {backend=}") + + metadata = TritonModelMetadata( + name=name, + backend=backend, + model=model, + tokenizer=self.tokenizer, + create_time=self.create_time, + request_converter=self._determine_request_converter(backend), ) + model_metadata[name] = metadata - async with self._metadata_lock: - if model_name not in self.model_metadata: - raise ClientError(f"Unknown model: {model_name}") - - # Blocking C API call dispatched to thread pool. The C API handles - # in-flight request draining and conflict resolution internally. - try: - await asyncio.to_thread(self._unload_model_sync, model_name) - except tritonserver.InvalidArgumentError as e: - raise ClientError(f"Failed to unload model '{model_name}': {e}") - except tritonserver.TritonError as e: - raise ServerError(f"Failed to unload model '{model_name}': {e}") - - del self.model_metadata[model_name] - - def _unload_model_sync(self, model_name: str) -> None: - self.server.unload(model_name) + return model_metadata def _get_streaming_chat_response_chunk( self, @@ -607,7 +272,6 @@ def _get_streaming_chat_response_chunk( request_id: str, created: int, model: str, - usage: Optional[CompletionUsage] = None, ) -> CreateChatCompletionStreamResponse: return CreateChatCompletionStreamResponse( id=request_id, @@ -616,7 +280,6 @@ def _get_streaming_chat_response_chunk( model=model, system_fingerprint=None, object=ObjectType.chat_completion_chunk, - usage=usage, ) def _get_first_streaming_chat_response( @@ -632,217 +295,55 @@ def _get_first_streaming_chat_response( finish_reason=None, ) chunk = self._get_streaming_chat_response_chunk( - choice, request_id, created, model, usage=None + choice, request_id, created, model + ) + return chunk + + def _get_nth_streaming_chat_response( + self, + request_id: str, + created: int, + model: str, + response: tritonserver.InferenceResponse, + ) -> CreateChatCompletionStreamResponse: + text = _get_output(response) + choice = ChatCompletionStreamingResponseChoice( + index=0, + delta=ChatCompletionStreamResponseDelta( + role=None, content=text, function_call=None + ), + logprobs=None, + finish_reason=ChatCompletionFinishReason.stop if response.final else None, + ) + + chunk = self._get_streaming_chat_response_chunk( + choice, request_id, created, model ) return chunk async def _streaming_chat_iterator( self, request_id: str, - backend: str, created: int, - request: CreateChatCompletionRequest, + model: str, role: str, - tool_call_parser: ToolCallParser, responses: AsyncIterable, ) -> AsyncIterator[str]: - model = request.model - - tool_function_name = self._get_named_function_name(request=request) - - # Determine whether tools are in use with "auto" tool choice - tool_choice_auto = ( - tool_call_parser - and not tool_function_name - and self._should_stream_with_auto_tool_parsing(request) - ) - - previous_text = "" - include_usage = request.stream_options and request.stream_options.include_usage - usage_accumulator = _StreamingUsageAccumulator(backend) - chunk = self._get_first_streaming_chat_response( request_id, created, model, role ) yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n" async for response in responses: - delta_text = _get_output(response) - if include_usage: - usage_accumulator.update(response) - - ( - response_delta, - finish_reason, - current_text, - ) = self._get_streaming_response_delta( - previous_text=previous_text, - delta_text=delta_text, - tool_function_name=tool_function_name, - tool_choice_auto=tool_choice_auto, - tool_call_parser=tool_call_parser, - backend=backend, - is_final_response=response.final, - ) - previous_text = current_text - - # Parse logprobs for this chunk if requested - chunk_logprobs = None - if request.logprobs: - openai_logprobs = _get_openai_chat_format_logprobs_from_vllm_response( - response - ) - if openai_logprobs: - chunk_logprobs = ChatCompletionLogprobs(content=openai_logprobs) - - # if the response delta is None (e.g. because it was a - # "control token" for tool calls or the parser otherwise - # wasn't ready to send a token, then - # get the next token without streaming a chunk - if response_delta is None and finish_reason is None: - continue - - if finish_reason and response_delta is None: - response_delta = ChatCompletionStreamResponseDelta(content="") - - choice = ChatCompletionStreamingResponseChoice( - index=0, - delta=response_delta, - logprobs=chunk_logprobs, - finish_reason=finish_reason, - ) - - chunk = self._get_streaming_chat_response_chunk( - choice, request_id, created, model, usage=None + chunk = self._get_nth_streaming_chat_response( + request_id, created, model, response ) yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n" - # Send the final usage chunk if requested via stream_options. - if include_usage: - usage_payload = usage_accumulator.get_final_usage() - if usage_payload: - final_usage_chunk = CreateChatCompletionStreamResponse( - id=request_id, - choices=[], - created=created, - model=model, - system_fingerprint=None, - object=ObjectType.chat_completion_chunk, - usage=usage_payload, - ) - yield f"data: {final_usage_chunk.model_dump_json(exclude_unset=True)}\n\n" - yield "data: [DONE]\n\n" - def _get_streaming_response_delta( - self, - previous_text: str, - delta_text: str, - tool_function_name: Optional[str], - tool_choice_auto: bool, - tool_call_parser: ToolCallParser, - backend: str, - is_final_response: bool, - ) -> Tuple[ - Optional[ChatCompletionStreamResponseDelta], - Optional[ChatCompletionFinishReason], - str, - ]: - response_delta: Optional[ChatCompletionStreamResponseDelta] - current_text = "" - if tool_function_name: - response_delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=0, - function=Function2( - name=tool_function_name, arguments=delta_text - ), - ) - ] - ) - elif tool_choice_auto: - current_text = previous_text + delta_text - response_delta = tool_call_parser.parse_tool_calls_streaming( - current_text=current_text, delta_text=delta_text, backend=backend - ) - else: - response_delta = ChatCompletionStreamResponseDelta( - role=None, content=delta_text, function_call=None - ) - - if is_final_response: - auto_tools_called = False - if tool_call_parser: - auto_tools_called = len(tool_call_parser.prev_tool_call_arr) > 0 - index = ( - len(tool_call_parser.prev_tool_call_arr) - 1 - if auto_tools_called - else 0 - ) - else: - index = 0 - - # check to make sure we haven't "forgotten" to stream - # any tokens that were generated but previously - # matched by partial json parsing, such as '}'. - # only happens if we are NOT using structured outputs - # or guided decoding - if ( - self._should_check_for_unstreamed_tool_arg_tokens( - response_delta=response_delta, - auto_tools_called=auto_tools_called, - ) - and tool_call_parser - ): - latest_delta_len = 0 - if ( - isinstance(response_delta.tool_calls[0].function, Function2) - ) and isinstance(response_delta.tool_calls[0].function.arguments, str): - latest_delta_len = len( - response_delta.tool_calls[0].function.arguments - ) - # get the expected call based on partial JSON - # parsing which "autocompletes" the JSON - expected_call = json.dumps( - tool_call_parser.prev_tool_call_arr[index].get("arguments", {}), - ensure_ascii=False, - ) - # get what we've streamed so far for arguments - # for the current tool - actual_call = tool_call_parser.streamed_args_for_tool[index] - if latest_delta_len > 0: - actual_call = actual_call[:-latest_delta_len] - - # check to see if there's anything left to stream - remaining_call = expected_call.replace(actual_call, "", 1) - - response_delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=index, - function=Function2(arguments=remaining_call).model_dump( - exclude_none=True - ), - ) - ] - ) - - finish_reason = ( - ChatCompletionFinishReason.tool_calls - if auto_tools_called - else ChatCompletionFinishReason.stop - ) - else: - finish_reason = None - - return response_delta, finish_reason, current_text - def _validate_chat_request( - self, - request: CreateChatCompletionRequest, - metadata: TritonModelMetadata, - lora_name: str | None, + self, request: CreateChatCompletionRequest, metadata: TritonModelMetadata ): """ Validates a chat request to align with currently supported features. @@ -850,131 +351,35 @@ def _validate_chat_request( # Reject missing internal information needed to do inference if not metadata: - raise ClientError(f"Unknown model: {request.model}") + raise Exception(f"Unknown model: {request.model}") if not metadata.tokenizer: - raise ServerError("Unknown tokenizer") + raise Exception("Unknown tokenizer") if not metadata.backend: - raise ServerError("Unknown backend") + raise Exception("Unknown backend") - if not metadata.inference_request_converter: - raise ServerError( - f"Unknown inference request format for model: {request.model}" - ) - - if not metadata.embedding_request_converter: - raise ServerError( - f"Unknown embedding request format for model: {request.model}" - ) - - if ( - metadata.lora_configs is not None - and lora_name is not None - and lora_name - not in [lora_config.name for lora_config in metadata.lora_configs] - ): - raise ClientError(f"Unknown LoRA: {lora_name}; for model: {request.model}") + if not metadata.request_converter: + raise Exception(f"Unknown request format for model: {request.model}") # Reject unsupported features if requested if request.n and request.n > 1: - raise ClientError( + raise Exception( f"Received n={request.n}, but only single choice (n=1) is currently supported" ) - if request.logit_bias is not None: - raise ClientError("logit bias is not currently supported") - - # Logprobs are only supported for vLLM backend currently - if metadata.backend != "vllm" and ( - request.logprobs or request.top_logprobs is not None - ): - raise ClientError( - "logprobs are currently available only for the vLLM backend" - ) - - if request.top_logprobs is not None and not request.logprobs: - raise ClientError("`top_logprobs` can only be used when `logprobs` is True") - - self._verify_chat_tool_call_settings(request=request) - - if request.stream_options and not request.stream: - raise ClientError("`stream_options` can only be used when `stream` is True") - - def _verify_chat_tool_call_settings(self, request: CreateChatCompletionRequest): - if ( - request.tool_choice - and request.tool_choice.root == ChatCompletionToolChoiceOption1.required - and not request.tools - ): - raise ClientError( - '"required" tool choice requires CreateChatCompletionRequest.tools to be provided' - ) - - if ( - request.tool_choice - and isinstance(request.tool_choice.root, ChatCompletionNamedToolChoice) - and not request.tools - ): - raise ClientError( - "Named tool choice requires CreateChatCompletionRequest.tools to be provided" - ) - - if ( - request.tool_choice - and request.tool_choice.root == ChatCompletionToolChoiceOption1.auto - and self.tool_call_parser is None - ): - raise ClientError( - '"auto" tool choice requires --tool-call-parser to be set' - ) - - if ( - request.tool_choice is None - and request.tools - and self.tool_call_parser is None - ): - raise ClientError( - "having tools in the request requires --tool-call-parser to be set" - ) + if request.logit_bias is not None or request.logprobs: + raise Exception("logit bias and log probs not currently supported") async def _streaming_completion_iterator( - self, - request_id: str, - created: int, - request: CreateCompletionRequest, - responses: AsyncIterable, - backend: str, + self, request_id: str, created: int, model: str, responses: AsyncIterable ) -> AsyncIterator[str]: - model = request.model - include_usage = request.stream_options and request.stream_options.include_usage - usage_accumulator = _StreamingUsageAccumulator(backend) - current_offset = 0 - async for response in responses: - if include_usage: - usage_accumulator.update(response) - text = _get_output(response) - - # Parse logprobs for this chunk if requested - chunk_logprobs = None - if request.logprobs is not None and request.logprobs > 0: - chunk_logprobs = ( - _get_openai_completion_format_logprobs_from_vllm_response(response) - ) - # Adjust text offsets based on accumulated output - if chunk_logprobs and chunk_logprobs.text_offset: - chunk_logprobs.text_offset = [ - offset + current_offset for offset in chunk_logprobs.text_offset - ] - - current_offset += len(text) - choice = Choice( finish_reason=FinishReason.stop if response.final else None, index=0, - logprobs=chunk_logprobs, + logprobs=None, text=text, ) chunk = CreateCompletionResponse( @@ -984,180 +389,48 @@ async def _streaming_completion_iterator( object=ObjectType.text_completion, created=created, model=model, - usage=None, ) yield f"data: {chunk.model_dump_json(exclude_unset=True)}\n\n" - # Send the final usage chunk if requested via stream_options. - if include_usage: - usage_payload = usage_accumulator.get_final_usage() - if usage_payload: - final_usage_chunk = CreateCompletionResponse( - id=request_id, - choices=[], - system_fingerprint=None, - object=ObjectType.text_completion, - created=created, - model=model, - usage=usage_payload, - ) - yield f"data: {final_usage_chunk.model_dump_json(exclude_unset=True)}\n\n" - yield "data: [DONE]\n\n" def _validate_completion_request( - self, - request: CreateCompletionRequest, - metadata: TritonModelMetadata, - lora_name: str | None, + self, request: CreateCompletionRequest, metadata: TritonModelMetadata ): """ Validates a completions request to align with currently supported features. """ # Reject missing internal information needed to do inference if not metadata: - raise ClientError(f"Unknown model: {request.model}") + raise Exception(f"Unknown model: {request.model}") if not metadata.backend: - raise ServerError("Unknown backend") - - if not metadata.inference_request_converter: - raise ServerError( - f"Unknown inference request format for model: {request.model}" - ) - - if not metadata.embedding_request_converter: - raise ServerError( - f"Unknown embedding request format for model: {request.model}" - ) + raise Exception("Unknown backend") - if ( - metadata.lora_configs is not None - and lora_name is not None - and lora_name - not in [lora_config.name for lora_config in metadata.lora_configs] - ): - raise ClientError(f"Unknown LoRA: {lora_name}; for model: {request.model}") + if not metadata.request_converter: + raise Exception(f"Unknown request format for model: {request.model}") # Reject unsupported features if requested if request.suffix is not None: - raise ClientError("suffix is not currently supported") + raise Exception("suffix is not currently supported") if not request.prompt: - raise ClientError("prompt must be non-empty") + raise Exception("prompt must be non-empty") # Currently only support single string as input if not isinstance(request.prompt, str): - raise ClientError("only single string input is supported") - - if "best_of" in request.model_fields_set and metadata.backend == "vllm": - raise ClientError( - "best_of is no longer supported in vLLM backend, removed from vLLM V1 engine" - ) + raise Exception("only single string input is supported") if request.n and request.n > 1: - raise ClientError( + raise Exception( f"Received n={request.n}, but only single choice (n=1) is currently supported" ) if request.best_of and request.best_of > 1: - raise ClientError( + raise Exception( f"Received best_of={request.best_of}, but only single choice (best_of=1) is currently supported" ) - if request.logit_bias is not None: - raise ClientError("logit bias is not supported") - - # Logprobs are only supported for vLLM backend currently - if ( - request.logprobs is not None - and request.logprobs > 0 - and metadata.backend != "vllm" - ): - raise ClientError( - "logprobs are currently available only for the vLLM backend" - ) - - if request.stream_options and not request.stream: - raise ClientError("`stream_options` can only be used when `stream` is True") - - def _validate_embedding_request( - self, - request: CreateEmbeddingRequest, - metadata: TritonModelMetadata, - ): - """ - Validates an embedding request to align with currently supported features. - """ - - # Reject missing internal information needed to do inference - if not metadata: - raise ClientError(f"Unknown model: {request.model}") - - if not metadata.backend: - raise ServerError("Unknown backend") - - if not metadata.inference_request_converter: - raise ServerError( - f"Unknown inference request format for model: {request.model}" - ) - - if not metadata.embedding_request_converter: - raise ServerError( - f"Unknown embedding request format for model: {request.model}" - ) - - def _should_stream_with_auto_tool_parsing( - self, request: CreateChatCompletionRequest - ): - has_tools = request.tools and self.tool_call_parser - auto_tool = ( - request.tool_choice is None - or request.tool_choice.root == ChatCompletionToolChoiceOption1.auto - ) - return has_tools and auto_tool - - def _should_check_for_unstreamed_tool_arg_tokens( - self, response_delta: ChatCompletionStreamResponseDelta, auto_tools_called - ): - return bool( - auto_tools_called - and self.tool_call_parser - and response_delta - and response_delta.tool_calls - and response_delta.tool_calls[0] - and response_delta.tool_calls[0].function - and response_delta.tool_calls[0].function.arguments is not None - ) - - def _get_named_function_name( - self, request: CreateChatCompletionRequest - ) -> Optional[str]: - if request.tool_choice and isinstance( - request.tool_choice.root, ChatCompletionNamedToolChoice - ): - tool_choice_function_name = request.tool_choice.root.function.name - else: - tool_choice_function_name = None - - if ( - request.tool_choice - and request.tool_choice.root == ChatCompletionToolChoiceOption1.required - ): - tool_choice_required_function_name = request.tools[0].function.name - else: - tool_choice_required_function_name = None - - return tool_choice_function_name or tool_choice_required_function_name - - def _get_lora_config( - self, model_name: str, lora_name: Optional[str] - ) -> TritonLoraConfig: - model_metadata = self.model_metadata.get(model_name) - if lora_name is None or model_metadata.lora_configs is None: - return None - for lora_config in model_metadata.lora_configs: - if lora_config.name == lora_name: - return lora_config - raise ClientError(f"Unknown LoRA: {lora_name}; for model: {model_name}") + if request.logit_bias is not None or request.logprobs is not None: + raise Exception("logit bias and log probs not supported") diff --git a/python/openai/openai_frontend/engine/utils/chat.py b/python/openai/openai_frontend/engine/utils/chat.py deleted file mode 100644 index 2c941e666c..0000000000 --- a/python/openai/openai_frontend/engine/utils/chat.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json -from typing import Dict, Iterable, List, Optional, Required, TypedDict, Union, cast - -# FIXME: Converge on single set of types in either schemas.openai or openai.types -from openai.types.chat import ChatCompletionMessageToolCallParam -from openai.types.chat.chat_completion_message_tool_call_param import Function -from schemas.openai import ( - ChatCompletionMessageToolCall, - ChatCompletionRequestAssistantMessage, - ChatCompletionRequestMessage, - ChatCompletionRequestMessageContentPart, - ChatCompletionRequestToolMessage, - ChatCompletionRequestUserMessage, - Type1, -) -from utils.utils import ClientError - - -class ConversationMessage(TypedDict, total=False): - role: Required[str] - """The role of the message's author.""" - - content: Union[Optional[str], List[Dict[str, str]]] - """The contents of the message""" - - tool_call_id: Optional[str] - """Tool call that this message is responding to.""" - - name: Optional[str] - """The name of the function to call""" - - tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]] - """The tool calls generated by the model, such as function calls.""" - - -def _frontend_schema_to_openai_schema_completion_tool_call( - tool_call_param: ChatCompletionMessageToolCall, -) -> ChatCompletionMessageToolCallParam: - return ChatCompletionMessageToolCallParam( - id=tool_call_param.id, - type=tool_call_param.type, - function=Function( - name=tool_call_param.function.name, - arguments=tool_call_param.function.arguments, - ), - ) - - -def _parse_chat_message_content_parts( - role: str, parts: List[ChatCompletionRequestMessageContentPart] -) -> ConversationMessage: - content = list[Dict]() - - for part in parts: - if part.root.type == Type1.text or part.root.type == "text": - parse_res = {"type": "text", "text": part.root.text} - content.append(parse_res) - else: - raise ClientError( - f"only text message is supported, but got {part.root.type}" - ) - - return ConversationMessage(role=role, content=content) - - -def _parse_chat_message_content( - message: ChatCompletionRequestMessage, -) -> ConversationMessage: - role = message.root.role - content = message.root.content - - if content is None or isinstance(content, str): - result_msg = ConversationMessage(role=role, content=content) - else: # content is a list of message parts - result_msg = _parse_chat_message_content_parts( - role, - content, - ) - - if role == "assistant": - parsed_msg = cast(ChatCompletionRequestAssistantMessage, message.root) - - if parsed_msg.tool_calls: - result_msg["tool_calls"] = list( - [ - _frontend_schema_to_openai_schema_completion_tool_call(tool_call) - for tool_call in parsed_msg.tool_calls.root - ] - ) - elif role == "tool": - parsed_msg = cast(ChatCompletionRequestToolMessage, message.root) - if parsed_msg.tool_call_id: - result_msg["tool_call_id"] = parsed_msg.tool_call_id - - if isinstance(message.root, ChatCompletionRequestUserMessage) and isinstance( - message.root.name, str - ): - result_msg["name"] = message.root.name - - return result_msg - - -def _postprocess_messages(messages: List[ConversationMessage]) -> None: - # per the Transformers docs & maintainers, tool call arguments in - # assistant-role messages with tool_calls need to be dicts not JSON str - - # this is how tool-use chat templates will expect them moving forwards - # so, for messages that have tool_calls, parse the string (which we get - # from openAI format) to dict - for message in messages: - if ( - message["role"] == "assistant" - and "tool_calls" in message - and isinstance(message["tool_calls"], list) - ): - for item in message["tool_calls"]: - item["function"]["arguments"] = json.loads( - item["function"]["arguments"] - ) - - -def parse_chat_messages( - messages: List[ChatCompletionRequestMessage], -) -> List[ConversationMessage]: - conversation: List[ConversationMessage] = [] - - for msg in messages: - sub_message = _parse_chat_message_content(msg) - conversation.append(sub_message) - - _postprocess_messages(conversation) - - return conversation - - -# This function loads the chat template file content -# if the user chooses to use a chat template different from -# the original one provided with the model's tokenizer. -def load_chat_template(chat_template) -> Optional[str]: - if chat_template is None: - return None - - with open(chat_template) as f: - return f.read() diff --git a/python/openai/openai_frontend/engine/utils/tokenizer.py b/python/openai/openai_frontend/engine/utils/tokenizer.py index 73184e843f..982e553cea 100644 --- a/python/openai/openai_frontend/engine/utils/tokenizer.py +++ b/python/openai/openai_frontend/engine/utils/tokenizer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,8 +32,6 @@ from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast -AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast] - def get_cached_tokenizer( tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast] @@ -47,6 +45,7 @@ def get_cached_tokenizer( function caches these properties for faster access.""" tokenizer_all_special_ids = set(tokenizer.all_special_ids) + tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended tokenizer_all_special_tokens = set(tokenizer.all_special_tokens) tokenizer_len = len(tokenizer) @@ -59,6 +58,10 @@ def all_special_ids(self): def all_special_tokens(self): return tokenizer_all_special_tokens + @property + def all_special_tokens_extended(self): + return tokenizer_all_special_tokens_extended + def __len__(self): return tokenizer_len diff --git a/python/openai/openai_frontend/engine/utils/tool_call_parsers/__init__.py b/python/openai/openai_frontend/engine/utils/tool_call_parsers/__init__.py deleted file mode 100644 index d4bec1e62f..0000000000 --- a/python/openai/openai_frontend/engine/utils/tool_call_parsers/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Adapted from -# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/tool_parsers/__init__.py -# Copyright 2024 The vLLM team. - -from .llama_tool_call_parser import Llama3JsonToolParser -from .mistral_tool_call_parser import MistralToolParser -from .tool_call_parser import ToolCallParser, ToolParserManager - -__all__ = [ - "ToolCallParser", - "ToolParserManager", - "Llama3JsonToolParser", - "MistralToolParser", -] diff --git a/python/openai/openai_frontend/engine/utils/tool_call_parsers/llama_tool_call_parser.py b/python/openai/openai_frontend/engine/utils/tool_call_parsers/llama_tool_call_parser.py deleted file mode 100644 index ba63189707..0000000000 --- a/python/openai/openai_frontend/engine/utils/tool_call_parsers/llama_tool_call_parser.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Adapted from -# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py -# Copyright 2024 The vLLM team. -import json -import uuid -from typing import Union - -import partial_json_parser -from engine.utils.tokenizer import AnyTokenizer -from engine.utils.tool_call_parsers.tool_call_parser import ( - ToolCallParser, - ToolParserManager, -) -from partial_json_parser.core.options import Allow -from schemas.openai import ( - ChatCompletionMessageToolCall, - ChatCompletionMessageToolCallChunk, - ChatCompletionMessageToolCalls, - ChatCompletionResponseMessage, - ChatCompletionStreamResponseDelta, - Function1, - Function2, -) - -from .utils import find_common_prefix, is_complete_json, partial_json_loads - - -@ToolParserManager.register_module("llama3") -class Llama3JsonToolParser(ToolCallParser): - def __init__(self, tokenizer: AnyTokenizer): - super().__init__(tokenizer) - - # initialize properties used for state when parsing tool calls in - # streaming mode - self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list - - self.bot_token = "<|python_tag|>" - - def parse_tool_calls( - self, full_text: str, role: str, backend: str - ) -> ChatCompletionResponseMessage: - """ - Extract the tool calls from a complete model response. - """ - # case -- if a tool call token is not present, return a text response - if not (full_text.startswith(self.bot_token) or full_text.startswith("{")): - return ChatCompletionResponseMessage( - tool_calls=None, content=full_text, role=role - ) - - original_full_text = full_text - try: - # FIXME: tensorrt_llm backend might generate some unnecessary text messages - # after the tool call json text starting with "assistant\n\n". - if backend != "vllm": - last_index = full_text.find("assistant\n\n") - if last_index > 0: - full_text = full_text[:last_index] - - # load the JSON, and then use it to build the Function and - # Tool Call - dec = json.JSONDecoder() - function_call_arr = [] - - # depending on the prompt format the Llama model may or may not - # prefix the output with the <|python_tag|> token - start_idx = ( - len(self.bot_token) if full_text.startswith(self.bot_token) else 0 - ) - while start_idx < len(full_text): - (obj, end_idx) = dec.raw_decode(full_text[start_idx:]) - start_idx += end_idx + len("; ") - function_call_arr.append(obj) - - tool_calls = ChatCompletionMessageToolCalls( - root=[ - ChatCompletionMessageToolCall( - id=f"cmpl-{uuid.uuid1()}", - type="function", - function=Function1( - name=raw_function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - raw_function_call["arguments"] - if "arguments" in raw_function_call - else raw_function_call["parameters"] - ), - ), - ) - for raw_function_call in function_call_arr - ] - ) - - # get any content before the tool call - ret = ChatCompletionResponseMessage( - tool_calls=tool_calls, content="", role=role - ) - return ret - - except Exception as e: - # return information to just treat the tool call as regular JSON - return ChatCompletionResponseMessage( - tool_calls=None, content=original_full_text, role=role - ) - - def parse_tool_calls_streaming( - self, current_text: str, delta_text: str, backend: str - ) -> Union[ChatCompletionStreamResponseDelta, None]: - if not ( - current_text.startswith(self.bot_token) or current_text.startswith("{") - ): - return ChatCompletionStreamResponseDelta(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - try: - tool_call_arr = [] - is_complete = [] - try: - # depending on the prompt format the Llama model may or may not - # prefix the output with the <|python_tag|> token - start_idx = ( - len(self.bot_token) - if current_text.startswith(self.bot_token) - else 0 - ) - while start_idx < len(current_text): - (obj, end_idx) = partial_json_loads(current_text[start_idx:], flags) - is_complete.append( - is_complete_json(current_text[start_idx : start_idx + end_idx]) - ) - start_idx += end_idx + len("; ") - # depending on the prompt Llama can use - # either arguments or parameters - if "parameters" in obj: - assert ( - "arguments" not in obj - ), "model generated both parameters and arguments" - obj["arguments"] = obj["parameters"] - tool_call_arr.append(obj) - except partial_json_parser.core.exceptions.MalformedJSON: - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get("arguments") - if cur_arguments: - cur_args_json = json.dumps(cur_arguments) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - function=Function2( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[ - self.current_tool_id - ] += argument_diff - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - type="function", - id=f"cmpl-{uuid.uuid1()}", - function=Function2(name=function_name).model_dump( - exclude_none=True - ), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get("arguments") - delta = None - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - function=Function2( - arguments=argument_diff - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[ - self.current_tool_id - ] += argument_diff - - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception: - return None diff --git a/python/openai/openai_frontend/engine/utils/tool_call_parsers/mistral_tool_call_parser.py b/python/openai/openai_frontend/engine/utils/tool_call_parsers/mistral_tool_call_parser.py deleted file mode 100644 index e3af3944e9..0000000000 --- a/python/openai/openai_frontend/engine/utils/tool_call_parsers/mistral_tool_call_parser.py +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Adapted from -# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py -# Copyright 2024 The vLLM team. -import json -import re -from random import choices -from string import ascii_letters, digits -from typing import Dict, List, Union - -import partial_json_parser -from engine.utils.tokenizer import AnyTokenizer -from engine.utils.tool_call_parsers.tool_call_parser import ( - ToolCallParser, - ToolParserManager, -) -from partial_json_parser.core.options import Allow -from schemas.openai import ( - ChatCompletionMessageToolCall, - ChatCompletionMessageToolCallChunk, - ChatCompletionMessageToolCalls, - ChatCompletionResponseMessage, - ChatCompletionStreamResponseDelta, - Function1, - Function2, -) - -from .utils import extract_intermediate_diff - -ALPHANUMERIC = ascii_letters + digits - - -def generate_mistral_random_id(): - # Mistral Tool Call Ids must be alphanumeric with a maximum length of 9. - # https://github.com/mistralai/mistral-common/blob/21ee9f6cee3441e9bb1e6ed2d10173f90bd9b94b/src/mistral_common/protocol/instruct/validator.py#L299 - return "".join(choices(ALPHANUMERIC, k=9)) - - -@ToolParserManager.register_module("mistral") -class MistralToolParser(ToolCallParser): - def __init__(self, tokenizer: AnyTokenizer): - super().__init__(tokenizer) - - # initialize properties used for state when parsing tool calls in - # streaming mode - self.prev_tool_call_arr: List[Dict] = [] - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: List[ - str - ] = [] # map what has been streamed for each tool so far to a list - self.bot_token = "[TOOL_CALLS]" - self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) - - def parse_tool_calls( - self, full_text: str, role: str, backend: str - ) -> ChatCompletionResponseMessage: - """ - Extract the tool calls from a complete model response. Requires - find-and-replacing single quotes with double quotes for JSON parsing, - make sure your tool call arguments don't ever include quotes! - """ - - # case -- if a tool call token is not present, return a text response - if not (full_text.startswith(self.bot_token) or full_text.startswith("[")): - return ChatCompletionResponseMessage( - tool_calls=None, content=full_text, role=role - ) - - # first remove the BOT token - tool_content = full_text.replace(self.bot_token, "").strip() - try: - # we first try to directly load the json as parsing very nested - # jsons is difficult - try: - function_call_arr = json.loads(tool_content) - except json.JSONDecodeError: - # use a regex to find the part corresponding to the tool call. - # NOTE: This use case should not happen if the model is trained - # correctly. It's a easy possible fix so it's included, but - # can be brittle for very complex / highly nested tool calls - raw_tool_call = self.tool_call_regex.findall(tool_content)[0] - function_call_arr = json.loads(raw_tool_call) - - # Tool Call - tool_calls = ChatCompletionMessageToolCalls( - root=[ - ChatCompletionMessageToolCall( - id=generate_mistral_random_id(), - type="function", - function=Function1( - name=raw_function_call["name"], - # function call args are JSON but as a string - arguments=json.dumps( - raw_function_call["arguments"], ensure_ascii=False - ), - ), - ) - for raw_function_call in function_call_arr - ] - ) - - # get any content before the tool call - content = ( - full_text.split(self.bot_token)[0] - if full_text.startswith(self.bot_token) - else "" - ) - return ChatCompletionResponseMessage( - tool_calls=tool_calls, content=content, role=role - ) - - except Exception: - # return information to just treat the tool call as regular JSON - return ChatCompletionResponseMessage( - tool_calls=None, content=full_text, role=role - ) - - def parse_tool_calls_streaming( - self, current_text: str, delta_text: str, backend: str - ) -> Union[ChatCompletionStreamResponseDelta, None]: - # if the tool call token is not in the tokens generated so far, append - # output to contents since it's not a tool - # tensorrt_llm backend likely doesn't generate the bos token - if not (self.bot_token in current_text or "[" in current_text): - return ChatCompletionStreamResponseDelta(content=delta_text) - - # handle if we detected the BOT token which means the start of tool - # calling - if self.bot_token == delta_text.strip(): - # if it's the only token, return None, so we don't send a chat - # completion any don't send a control token - return None - - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - - try: - # replace BOT token with empty string, and convert single quotes - # to double to allow parsing as JSON since mistral uses single - # quotes instead of double for tool calls - parsable_arr = current_text.split(self.bot_token)[-1] - - # tool calls are generated in an array, so do partial JSON - # parsing on the entire array - try: - tool_call_arr: List[Dict] = partial_json_parser.loads( - parsable_arr, flags - ) - except partial_json_parser.core.exceptions.MalformedJSON: - return None - - # select as the current tool call the one we're on the state at - - current_tool_call: Dict = ( - tool_call_arr[self.current_tool_id] if len(tool_call_arr) > 0 else {} - ) - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif ( - len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1 - ): - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - diff: Union[str, None] = current_tool_call.get("arguments") - - if diff: - diff = json.dumps(diff, ensure_ascii=False).replace( - self.streamed_args_for_tool[self.current_tool_id], "" - ) - - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - function=Function2(arguments=diff).model_dump( - exclude_none=True - ), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - return delta - - # case: update an existing tool - this is handled below - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = current_tool_call.get("name") - if function_name: - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - type="function", - id=generate_mistral_random_id(), - function=Function2(name=function_name).model_dump( - exclude_none=True - ), - ) - ] - ) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - cur_arguments = current_tool_call.get("arguments") - - new_text = delta_text.replace("'", '"') - if '"}' in new_text: - new_text = new_text[: new_text.rindex('"}')] - - if not cur_arguments and not prev_arguments: - delta = None - elif not cur_arguments and prev_arguments: - delta = None - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False)[ - :-2 - ] - - if new_text not in cur_arguments_json: - return None - arguments_delta = cur_arguments_json[ - : cur_arguments_json.rindex(new_text) + len(new_text) - ] - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - function=Function2( - arguments=arguments_delta - ).model_dump(exclude_none=True), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - - argument_diff = extract_intermediate_diff( - cur_args_json, prev_args_json - ) - delta = ChatCompletionStreamResponseDelta( - tool_calls=[ - ChatCompletionMessageToolCallChunk( - index=self.current_tool_id, - function=Function2(arguments=argument_diff).model_dump( - exclude_none=True - ), - ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - else: - # try parsing it with regular JSON - if it works we're - # at the end, and we need to send the difference between - # tokens streamed so far and the valid JSON - delta = None - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - self.prev_tool_call_arr = tool_call_arr - return delta - - except Exception: - return None diff --git a/python/openai/openai_frontend/engine/utils/tool_call_parsers/tool_call_parser.py b/python/openai/openai_frontend/engine/utils/tool_call_parsers/tool_call_parser.py deleted file mode 100644 index 5ae192eb6c..0000000000 --- a/python/openai/openai_frontend/engine/utils/tool_call_parsers/tool_call_parser.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Adapted from -# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py -# Copyright 2024 The vLLM team. -from typing import Callable, Dict, List, Optional, Union - -from engine.utils.tokenizer import AnyTokenizer -from schemas.openai import ( - ChatCompletionMessageToolCalls, - ChatCompletionStreamResponseDelta, -) - - -class ToolCallParser: - """The Base Tool Call Parser for parsing the Tool Call from the responses, - Two inferfaces are supported: the one-time parser for synchronized response - and streaming parser for streaming response. - """ - - def __init__(self, tokenizer: AnyTokenizer): - self.prev_tool_call_arr: List[Dict] = [] - # the index of the tool call that is currently being parsed - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: List[str] = [] - - self.model_tokenizer = tokenizer - - def parse_tool_calls( - self, full_text: str, role: str, backend: str - ) -> ChatCompletionMessageToolCalls: - raise NotImplementedError( - "BaseToolCallParser.parse_tool_calls has not been implemented!" - ) - - def parse_tool_calls_streaming( - self, current_text: str, delta_text: str, backend: str - ) -> ChatCompletionStreamResponseDelta: - raise NotImplementedError( - "BaseToolCallParser.parse_tool_calls_streaming has not been implemented!" - ) - - -class ToolParserManager: - tool_parsers: dict[str, type] = {} - - @classmethod - def get_tool_parser_cls(cls, name) -> type: - if name in cls.tool_parsers: - return cls.tool_parsers[name] - - raise KeyError(f"tool parser: '{name}' not found in tool_call_parsers") - - @classmethod - def _register_module( - cls, - module: type, - module_name: Optional[Union[str, list[str]]] = None, - force: bool = True, - ) -> None: - if not issubclass(module, ToolCallParser): - raise TypeError( - f"module must be subclass of ToolCallParser, but got {type(module)}" - ) - if module_name is None: - module_name = module.__name__ - if isinstance(module_name, str): - module_name = [module_name] - for name in module_name: - if not force and name in cls.tool_parsers: - existed_module = cls.tool_parsers[name] - raise KeyError( - f"{name} is already registered " f"at {existed_module.__module__}" - ) - cls.tool_parsers[name] = module - - @classmethod - def register_module( - cls, - name: Optional[Union[str, list[str]]] = None, - force: bool = True, - module: Union[type, None] = None, - ) -> Union[type, Callable]: - """ - Register module with the given name or name list. it can be used as a - decoder(with module as None) or normal function(with module as not - None). - """ - if not isinstance(force, bool): - raise TypeError(f"force must be a boolean, but got {type(force)}") - - # raise the error ahead of time - if not (name is None or isinstance(name, str)): - raise TypeError( - "name must be None, an instance of str, " f"but got {type(name)}" - ) - - # use it as a normal method: x.register_module(module=SomeClass) - if module is not None: - cls._register_module(module=module, module_name=name, force=force) - return module - - # use it as a decorator: @x.register_module() - def _register(module): - cls._register_module(module=module, module_name=name, force=force) - return module - - return _register diff --git a/python/openai/openai_frontend/engine/utils/tool_call_parsers/utils.py b/python/openai/openai_frontend/engine/utils/tool_call_parsers/utils.py deleted file mode 100644 index 60513d3f8f..0000000000 --- a/python/openai/openai_frontend/engine/utils/tool_call_parsers/utils.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Adapted from -# https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/tool_parsers/utils.py -# Copyright 2024 The vLLM team. - -import json -from typing import Any - -import partial_json_parser -from partial_json_parser.core.options import Allow - - -# partial_json_parser doesn't support extra data and -# JSONDecorder.raw_decode doesn't support partial JSON -def partial_json_loads(input_str: str, flags: Allow) -> tuple[Any, int]: - try: - return (partial_json_parser.loads(input_str, flags), len(input_str)) - except json.JSONDecodeError as e: - if "Extra data" in e.msg: - dec = json.JSONDecoder() - return dec.raw_decode(input_str) - raise - - -def is_complete_json(input_str: str) -> bool: - try: - json.loads(input_str) - return True - except json.JSONDecodeError: - return False - - -def find_common_prefix(s1: str, s2: str) -> str: - """ - Finds a common prefix that is shared between two strings, if there is one. - Order of arguments is NOT important. - - This function is provided as a UTILITY for extracting information from JSON - generated by partial_json_parser, to help in ensuring that the right tokens - are returned in streaming, so that close-quotes, close-brackets and - close-braces are not returned prematurely. - - e.g. find_common_prefix('{"fruit": "ap"}', '{"fruit": "apple"}') -> - '{"fruit": "ap' - """ - prefix = "" - min_length = min(len(s1), len(s2)) - for i in range(0, min_length): - if s1[i] == s2[i]: - prefix += s1[i] - else: - break - return prefix - - -def find_common_suffix(s1: str, s2: str) -> str: - """ - Finds a common suffix shared between two strings, if there is one. Order of - arguments is NOT important. - Stops when the suffix ends OR it hits an alphanumeric character - - e.g. find_common_suffix('{"fruit": "ap"}', '{"fruit": "apple"}') -> '"}' - """ - suffix = "" - min_length = min(len(s1), len(s2)) - for i in range(1, min_length + 1): - if s1[-i] == s2[-i] and not s1[-i].isalnum(): - suffix = s1[-i] + suffix - else: - break - return suffix - - -def extract_intermediate_diff(curr: str, old: str) -> str: - """ - Given two strings, extract the difference in the middle between two strings - that are known to have a common prefix and/or suffix. - - This function is provided as a UTILITY for extracting information from JSON - generated by partial_json_parser, to help in ensuring that the right tokens - are returned in streaming, so that close-quotes, close-brackets and - close-braces are not returned prematurely. The order of arguments IS - important - the new version of the partially-parsed JSON must be the first - argument, and the second argument must be from the previous generation. - - What it returns, is tokens that should be streamed to the client. - - e.g. extract_intermediate_diff('{"fruit": "apple"}', '{"fruit": "ap"}') - -> 'ple' - - """ - suffix = find_common_suffix(curr, old) - - old = old[::-1].replace(suffix[::-1], "", 1)[::-1] - prefix = find_common_prefix(curr, old) - diff = curr - if len(suffix): - diff = diff[::-1].replace(suffix[::-1], "", 1)[::-1] - - if len(prefix): - # replace the prefix only once in case it's mirrored - diff = diff.replace(prefix, "", 1) - - return diff diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 91925e16b0..2ec8cce7d5 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -23,57 +23,17 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import ctypes -import json -import os -import re -import sys -import traceback -from dataclasses import asdict, dataclass, field -from enum import Enum -from pathlib import Path -from typing import Dict, Iterable, List, Optional, Union +from typing import Iterable, List import numpy as np import tritonserver -from pydantic import BaseModel -from schemas.openai import ( - ChatCompletionNamedToolChoice, - ChatCompletionTokenLogprob, - ChatCompletionToolChoiceOption1, - CompletionUsage, - CreateChatCompletionRequest, - CreateCompletionRequest, - CreateEmbeddingRequest, - EmbeddingUsage, - Logprobs, - TopLogprob, -) -from utils.utils import ClientError, ServerError - - -class RequestKind(Enum): - GENERATION = 1 - EMBEDDING = 2 - +from schemas.openai import CreateChatCompletionRequest, CreateCompletionRequest -@dataclass -class TritonLoraConfig: - name: str - # Unique fields for TensorRT-LLM backend - task_id: Optional[int] = None - path: Optional[str] = None - is_registered: Optional[bool] = False - - -def _create_vllm_generate_request( - model, - prompt, - request: CreateChatCompletionRequest | CreateCompletionRequest, - lora_config: TritonLoraConfig | None, - echo_tensor_name: str | None, - default_max_tokens: int, +def _create_vllm_inference_request( + model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest ): inputs = {} # Exclude non-sampling parameters so they aren't passed to vLLM @@ -95,71 +55,15 @@ def _create_vllm_generate_request( "function_call", "functions", "suffix", - "max_completion_tokens", - # will be handled explicitly - "max_tokens", - "logprobs", - "top_logprobs", - # not supported for vLLM backend (removed from vLLM V1) but supported for TRT-LLM/Python backend - "best_of", } # NOTE: The exclude_none is important, as internals may not support # values of NoneType at this time. - sampling_parameters = request.model_dump( + sampling_parameters = request.model_dump_json( exclude=excludes, exclude_none=True, ) - request_logprobs = False - # Indicates CreateChatCompletionRequest - if hasattr(request, "max_completion_tokens"): - if request.max_completion_tokens is not None: - sampling_parameters["max_tokens"] = request.max_completion_tokens - # Fallback to deprecated request.max_tokens - elif request.max_tokens is not None: - sampling_parameters["max_tokens"] = request.max_tokens - # If neither is set, use a default value for max_tokens - else: - sampling_parameters["max_tokens"] = default_max_tokens - - # Handle logprobs for chat completions - # OpenAI API: logprobs (bool), top_logprobs (int 0-20) - # vLLM API: logprobs (int) - number of top token logprobs to return - if request.logprobs and request.top_logprobs is not None: - sampling_parameters["logprobs"] = request.top_logprobs - request_logprobs = True - elif request.logprobs: - # If logprobs=True but top_logprobs not specified, default to 1 - sampling_parameters["logprobs"] = 1 - request_logprobs = True - # Indicates CreateCompletionRequest - else: - if request.max_tokens is not None: - sampling_parameters["max_tokens"] = request.max_tokens - else: - sampling_parameters["max_tokens"] = default_max_tokens - - # Handle logprobs for completions - # OpenAI API: logprobs (int 0-5) - number of top token log probs - # vLLM API: logprobs (int) - same behavior, pass directly - if request.logprobs is not None and request.logprobs > 0: - sampling_parameters["logprobs"] = request.logprobs - request_logprobs = True - inputs["return_logprobs"] = np.bool_([request_logprobs]) - - if lora_config is not None: - sampling_parameters["lora_name"] = lora_config.name - - guided_json = _get_guided_json_from_tool(request) - if guided_json is not None: - from vllm.sampling_params import StructuredOutputsParams - - sampling_parameters["structured_outputs"] = json.dumps( - asdict(StructuredOutputsParams(json=guided_json)) - ) - sampling_parameters = json.dumps(sampling_parameters) - exclude_input_in_output = True echo = getattr(request, "echo", None) if echo is not None: @@ -167,43 +71,21 @@ def _create_vllm_generate_request( inputs["text_input"] = [prompt] inputs["stream"] = np.bool_([request.stream]) - inputs[echo_tensor_name] = np.bool_([exclude_input_in_output]) + inputs["exclude_input_in_output"] = np.bool_([exclude_input_in_output]) # Pass sampling_parameters as serialized JSON string input to support List # fields like 'stop' that aren't supported by TRITONSERVER_Parameters yet. inputs["sampling_parameters"] = [sampling_parameters] - inputs["return_num_input_tokens"] = np.bool_([True]) - inputs["return_num_output_tokens"] = np.bool_([True]) return model.create_request(inputs=inputs) -def _create_trtllm_generate_request( - model, - prompt, - request: CreateChatCompletionRequest | CreateCompletionRequest, - lora_config: TritonLoraConfig | None, - echo_tensor_name: str | None, - default_max_tokens: int, +def _create_trtllm_inference_request( + model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest ): inputs = {} inputs["text_input"] = [[prompt]] inputs["stream"] = np.bool_([[request.stream]]) - - # Indicates CreateChatCompletionRequest - if hasattr(request, "max_completion_tokens"): - if request.max_completion_tokens is not None: - inputs["max_tokens"] = np.int32([[request.max_completion_tokens]]) - # Fallback to deprecated request.max_tokens - elif request.max_tokens is not None: - inputs["max_tokens"] = np.int32([[request.max_tokens]]) - # If neither is set, use a default value for max_tokens - else: - inputs["max_tokens"] = np.int32([[default_max_tokens]]) - # Indicates CreateCompletionRequest - elif request.max_tokens is not None: + if request.max_tokens: inputs["max_tokens"] = np.int32([[request.max_tokens]]) - else: - inputs["max_tokens"] = np.int32([[default_max_tokens]]) - if request.stop: if isinstance(request.stop, str): request.stop = [request.stop] @@ -216,68 +98,14 @@ def _create_trtllm_generate_request( if request.presence_penalty is not None: inputs["presence_penalty"] = np.float32([[request.presence_penalty]]) if request.seed is not None: - inputs["seed"] = np.uint64([[request.seed]]) + inputs["random_seed"] = np.uint64([[request.seed]]) if request.temperature is not None: inputs["temperature"] = np.float32([[request.temperature]]) - # Only limited TRT-LLM models support "echo" (inflight_batcher_llm, disaggregated_serving, llmapi) - echo = getattr(request, "echo", None) - if echo is not None and echo_tensor_name is not None: - inputs[echo_tensor_name] = np.bool_([[not echo]]) - - guided_json = _get_guided_json_from_tool(request) - if guided_json is not None: - inputs["guided_decoding_guide_type"] = [["json_schema"]] - inputs["guided_decoding_guide"] = [[guided_json]] - - if lora_config is not None: - # To perform inference with a specific LoRA for the first time `lora_task_id` `lora_weights` and `lora_config` must all be given. - # The LoRA will be cached, so that subsequent requests for the same task only require `lora_task_id`. - inputs["lora_task_id"] = np.uint64([[lora_config.task_id]]) - if not lora_config.is_registered: - lora_weights_data = np.load( - os.path.join(lora_config.path, "model.lora_weights.npy") - ) - lora_config_data = np.load( - os.path.join(lora_config.path, "model.lora_config.npy") - ) - inputs["lora_weights"] = lora_weights_data - inputs["lora_config"] = lora_config_data - lora_config.is_registered = True - - inputs["return_num_input_tokens"] = np.bool_([[True]]) - inputs["return_num_output_tokens"] = np.bool_([[True]]) + # FIXME: TRT-LLM doesn't currently support runtime changes of 'echo' and it + # is configured at model load time, so we don't handle it here for now. return model.create_request(inputs=inputs) -def _create_vllm_embedding_request( - model, - request: CreateEmbeddingRequest, -): - inputs = {} - embedding_request = {} - embedding_request["input"] = request.input - - pooling_params = {} - dims = request.dimensions - if dims is not None: - pooling_params["dimensions"] = [dims] - embedding_request["pooling_params"] = pooling_params - - inputs["embedding_request"] = [json.dumps(embedding_request)] - inputs["return_num_input_tokens"] = np.bool_([True]) - inputs["return_num_output_tokens"] = np.bool_([True]) - return model.create_request(inputs=inputs) - - -def _create_trtllm_embedding_request( - model, - request: CreateEmbeddingRequest, -): - raise ClientError( - "TRT-LLM backend and Python backend do not support embedding requests" - ) - - def _construct_string_from_pointer(pointer: int, size: int) -> str: """Constructs a Python string from a C pointer and size.""" @@ -308,11 +136,11 @@ def _to_string(tensor: tritonserver.Tensor) -> str: # there is only a single string, so enforce it to avoid obscure errors. volume = _get_volume(tensor.shape) if volume != 1: - raise ServerError( + raise Exception( f"Expected to find 1 string in the output, found {volume} instead." ) if tensor.size < 4: - raise ServerError( + raise Exception( f"Expected string buffer to contain its serialized byte size, but found size of {tensor.size}." ) @@ -320,97 +148,6 @@ def _to_string(tensor: tritonserver.Tensor) -> str: return _construct_string_from_pointer(tensor.data_ptr + 4, tensor.size - 4) -@dataclass -class _StreamingUsageAccumulator: - """Helper class to accumulate token usage from a streaming response.""" - - backend: str - prompt_tokens: int = 0 - completion_tokens: int = 0 - _prompt_tokens_set: bool = field(init=False, default=False) - - def update(self, response: tritonserver.InferenceResponse): - """Extracts usage from a response and updates the token counts.""" - usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATION) - if usage: - # The prompt_tokens is received with every chunk but should only be set once. - if not self._prompt_tokens_set: - self.prompt_tokens = usage.prompt_tokens - self._prompt_tokens_set = True - self.completion_tokens += usage.completion_tokens - - def get_final_usage(self) -> Optional[CompletionUsage]: - """ - Returns the final populated CompletionUsage object if any tokens were tracked. - """ - # If _prompt_tokens_set is True, it means we have received and processed - # at least one valid usage payload. - if self._prompt_tokens_set: - return CompletionUsage( - prompt_tokens=self.prompt_tokens, - completion_tokens=self.completion_tokens, - total_tokens=self.prompt_tokens + self.completion_tokens, - ) - return None - - -def _get_usage_from_response( - response: tritonserver._api._response.InferenceResponse, - backend: str, - request_type: RequestKind, -) -> Optional[CompletionUsage | EmbeddingUsage]: - """ - Extracts token usage statistics from a Triton inference response. - """ - prompt_tokens = None - completion_tokens = None - - if ( - "num_input_tokens" in response.outputs - and "num_output_tokens" in response.outputs - ): - input_token_tensor = response.outputs["num_input_tokens"] - output_token_tensor = response.outputs["num_output_tokens"] - - if input_token_tensor.data_type == tritonserver.DataType.UINT32: - prompt_tokens_ptr = ctypes.cast( - input_token_tensor.data_ptr, ctypes.POINTER(ctypes.c_uint32) - ) - prompt_tokens = prompt_tokens_ptr[0] - elif input_token_tensor.data_type == tritonserver.DataType.INT32: - prompt_tokens_ptr = ctypes.cast( - input_token_tensor.data_ptr, ctypes.POINTER(ctypes.c_int32) - ) - prompt_tokens = prompt_tokens_ptr[0] - - if output_token_tensor.data_type == tritonserver.DataType.UINT32: - completion_tokens_ptr = ctypes.cast( - output_token_tensor.data_ptr, ctypes.POINTER(ctypes.c_uint32) - ) - completion_tokens = completion_tokens_ptr[0] - elif output_token_tensor.data_type == tritonserver.DataType.INT32: - completion_tokens_ptr = ctypes.cast( - output_token_tensor.data_ptr, ctypes.POINTER(ctypes.c_int32) - ) - completion_tokens = completion_tokens_ptr[0] - - if prompt_tokens is not None: - if request_type == RequestKind.GENERATION and completion_tokens is not None: - total_tokens = prompt_tokens + completion_tokens - return CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) - elif request_type == RequestKind.EMBEDDING: - return EmbeddingUsage( - prompt_tokens=prompt_tokens, - total_tokens=prompt_tokens, - ) - - return None - - # TODO: Use tritonserver.InferenceResponse when support is published def _get_output(response: tritonserver._api._response.InferenceResponse) -> str: if "text_output" in response.outputs: @@ -426,305 +163,13 @@ def _get_output(response: tritonserver._api._response.InferenceResponse) -> str: return "" -def _get_logprobs_from_response( - response: tritonserver._api._response.InferenceResponse, -) -> Optional[List[Dict]]: - """ - Extracts logprobs from a Triton inference response (vLLM backend). - - Returns: - List of dictionaries containing logprobs data, or None if not available. - Format: [ - { - token_id: { - "logprob": float, - "rank": int, - "decoded_token": str - } - }, - ... - ] - """ - if "logprobs" not in response.outputs: - return None - - logprobs_tensor = response.outputs["logprobs"] - if logprobs_tensor is None: - return None - - # The logprobs are stored as JSON string (vLLM backend) - logprobs_str = _to_string(logprobs_tensor) - - if logprobs_str == "null": - return None - - try: - logprobs_data = json.loads(logprobs_str) - return logprobs_data - except json.JSONDecodeError: - return None - - -def _get_openai_chat_format_logprobs_from_vllm_response( - response: tritonserver._api._response.InferenceResponse, -) -> Optional[List[ChatCompletionTokenLogprob]]: - """ - Convert logprobs from a Triton inference response (vLLM backend) to OpenAI chat completion format. - - Args: - response: Triton inference response containing logprobs output. - - Returns: - List of ChatCompletionTokenLogprob objects, or None if no logprobs available. - """ - vllm_logprobs = _get_logprobs_from_response(response) - - if not vllm_logprobs: - return None - - openai_logprobs = [] - for token_logprobs_dict in vllm_logprobs: - if not token_logprobs_dict: - continue - - # Sort by rank to identify the selected token (rank=1 is always the chosen token) - sorted_tokens = sorted( - token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) - ) - - # The first token (lowest rank) is the selected token - selected_token_id, selected_token_data = sorted_tokens[0] - selected_token = selected_token_data["decoded_token"] - selected_logprob = selected_token_data["logprob"] - - # Convert to bytes representation - token_bytes = list(selected_token.encode("utf-8")) - - top_logprobs_list = [] - for token_id, token_data in sorted_tokens: - decoded_token = token_data["decoded_token"] - top_logprobs_list.append( - TopLogprob( - token=decoded_token, - logprob=token_data["logprob"], - bytes=list(decoded_token.encode("utf-8")), - ) - ) - - openai_logprobs.append( - ChatCompletionTokenLogprob( - token=selected_token, - logprob=selected_logprob, - bytes=token_bytes, - top_logprobs=top_logprobs_list, - ) - ) - - return openai_logprobs - - -def _get_openai_completion_format_logprobs_from_vllm_response( - response: tritonserver._api._response.InferenceResponse, -) -> Optional[Logprobs]: - """ - Convert logprobs from a Triton inference response (vLLM backend) to OpenAI completion format. - - Args: - response: Triton inference response containing logprobs output. - - Returns: - Logprobs object for completions API, or None if no logprobs available. - """ - vllm_logprobs = _get_logprobs_from_response(response) - - if not vllm_logprobs: - return None - - text_offset = [] - token_logprobs = [] - tokens = [] - top_logprobs = [] - - current_offset = 0 - for token_logprobs_dict in vllm_logprobs: - if not token_logprobs_dict: - continue - - # Sort by rank to identify the selected token (rank=1 is always the chosen token) - sorted_tokens = sorted( - token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) - ) - - # The first token (lowest rank) is the selected token - selected_token_id, selected_token_data = sorted_tokens[0] - selected_token = selected_token_data["decoded_token"] - selected_logprob = selected_token_data["logprob"] - - text_offset.append(current_offset) - token_logprobs.append(selected_logprob) - tokens.append(selected_token) - - # Build top_logprobs dict for this position - top_logprobs_dict = {} - for token_id, token_data in sorted_tokens: - decoded_token = token_data["decoded_token"] - top_logprobs_dict[decoded_token] = token_data["logprob"] - top_logprobs.append(top_logprobs_dict) - - current_offset += len(selected_token) - - return Logprobs( - text_offset=text_offset, - token_logprobs=token_logprobs, - tokens=tokens, - top_logprobs=top_logprobs, - ) - - def _validate_triton_responses_non_streaming( responses: List[tritonserver._api._response.InferenceResponse], ): num_responses = len(responses) - if 1 <= num_responses <= 2: - if responses[-1].final != True: - raise ServerError("Unexpected internal error with incorrect response flags") - else: - raise ServerError( - f"Unexpected number of responses: {num_responses}, expected 1 or 2." - ) - - -def _get_guided_json_from_tool( - request: CreateChatCompletionRequest | CreateCompletionRequest, -) -> Optional[Union[str, dict, BaseModel]]: - if isinstance(request, CreateChatCompletionRequest): - if request.tool_choice is None or not request.tools: - return None - - if type(request.tool_choice.root) is ChatCompletionNamedToolChoice: - tool_name = request.tool_choice.root.function.name - elif request.tool_choice.root == ChatCompletionToolChoiceOption1.required: - tool_name = request.tools[0].function.name - else: - return None - - tools = {tool.function.name: tool.function for tool in request.tools} - if tool_name not in tools: - raise ClientError(f"Tool '{tool_name}' has not been passed in `tools`.") - tool = tools[tool_name] - return tool.parameters.model_dump_json() - - return None - - -def _validate_lora_path_trtllm(repo_path: str, lora_path: str, lora_name: str): - if os.path.isabs(lora_path): - raise ValueError( - f"LoRA path '{lora_path}' for '{lora_name}' must be a relative path inside its model repository" - ) - - # NOTE: Error messages should never contain the real/absolute paths. - realpath_repo = os.path.realpath(repo_path) - realpath_lora = os.path.realpath(os.path.join(realpath_repo, lora_path)) - # Always check if the LoRA path is inside the model repository before checking its existence. - if os.path.commonpath([realpath_repo, realpath_lora]) != realpath_repo: - raise ValueError( - f"LoRA path '{lora_path}' for '{lora_name}' must be inside its model repository" - ) - if not os.path.exists(realpath_lora): - raise ServerError( - f"LoRA directory '{lora_path}' not found for '{lora_name}' in its model repository" - ) - - # Check if the files exist - for lora_file in ["model.lora_weights.npy", "model.lora_config.npy"]: - lora_file_path = os.path.join(realpath_lora, lora_file) - if not os.path.exists(lora_file_path): - raise ServerError( - f"LoRA file '{lora_file}' not found for '{lora_name}' at path: {lora_file_path}" - ) - - -def _parse_lora_configs( - model_repository: str | list[str], model_name: str, model_version: int, backend: str -) -> None | List[tuple[str, str]]: - if ( - len(model_name) == 0 - or model_name.isspace() - or "/" in model_name - or "\\" in model_name - ): - raise ValueError( - f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." - ) - - lora_configs = [] - lora_task_id = 1 - repo_paths = model_repository - if isinstance(repo_paths, str): - repo_paths = [repo_paths] - for repo_path in repo_paths: - model_path = os.path.join(repo_path, model_name) - if (not Path(model_path).is_relative_to(repo_path)) or ( - os.path.normpath(model_path) != model_path - ): - raise ValueError( - f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." - ) - - model_path = os.path.normpath(model_path) - if not os.path.isdir(model_path): - # Cloud path? - return None - if model_version <= 0: - for version_path in os.listdir(model_path): - version = os.path.basename(version_path) - if re.fullmatch(r"^[0-9]+$", version) is None: - continue - model_version = max(model_version, int(version)) - if model_version <= 0: - # Model directory is malformed? - return None - version_path = os.path.join(model_path, str(model_version)) - lora_config_path = os.path.join(version_path, "multi_lora.json") - - if backend == "vllm": - is_lora_enabled = False - model_file_path = os.path.join(version_path, "model.json") - try: - with open(model_file_path, "r") as f: - config = json.load(f) - if "enable_lora" in config: - # The value could be a string or a bool. - is_lora_enabled = str(config["enable_lora"]).lower() == "true" - except Exception: - # Model directory or model.json is malformed? - return None - if is_lora_enabled != True: - continue - else: - # TRT-LLM backend does not use model.json - if not os.path.exists(lora_config_path): - continue - - try: - with open(lora_config_path, "r") as f: - lora_config = json.load(f) - for lora_name, lora_path in lora_config.items(): - if backend == "vllm": - lora_configs.append(TritonLoraConfig(name=lora_name)) - else: - _validate_lora_path_trtllm(repo_path, lora_path, lora_name) - lora_configs.append( - TritonLoraConfig( - name=lora_name, path=lora_path, task_id=lora_task_id - ) - ) - lora_task_id += 1 - except ServerError as e: - raise e - except Exception as e: - # LoRA is enabled but its list is not provided or malformed? - print(traceback.format_exc()) - return None - return lora_configs + if num_responses == 1 and responses[0].final != True: + raise Exception("Unexpected internal error with incorrect response flags") + if num_responses == 2 and responses[-1].final != True: + raise Exception("Unexpected internal error with incorrect response flags") + if num_responses > 2: + raise Exception(f"Unexpected number of responses: {num_responses}, expected 1.") diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py b/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py deleted file mode 100644 index f3dec540e2..0000000000 --- a/python/openai/openai_frontend/frontend/fastapi/middleware/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py deleted file mode 100644 index 085434ad46..0000000000 --- a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py +++ /dev/null @@ -1,229 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from fastapi import Request -from fastapi.responses import JSONResponse -from starlette.middleware.base import BaseHTTPMiddleware -from utils.utils import StatusCode - -# Mapping of API to their corresponding HTTP endpoints -ENDPOINT_MAPPING = { - "inference": [ - "POST /v1/chat/completions", - "POST /v1/completions", - "POST /v1/embeddings", - ], - "model-repository": [ - "GET /v1/models", - "POST /v1/models/", - ], - "metrics": ["GET /metrics"], - "health": ["GET /health/ready"], -} - - -class RestrictedFeatures: - """ - Manages API endpoint restrictions and their authentication requirements. - - This class parses command-line arguments for restricted API configurations - and provides methods to check if specific APIs are restricted - and what authentication is required. - """ - - def __init__(self, args: list[str]): - """ - Initialize the RestrictedFeatures with command-line arguments. - - Args: - args: List of --openai-restricted-api argument strings - (e.g., [["inference", "infer-key", "infer-value"], - ["model-repository", "model-key", "model-value"]]) - """ - self._restrictions = {} - self.ParseRestrictedFeatureOption(args) - - def ParseRestrictedFeatureOption(self, args): - """ - Parse command-line arguments to extract API restrictions. - - Args: - args: List of restriction configuration strings - - Raises: - ValueError: If unknown API is specified or duplicate API configs are found - """ - for apis, key, value in args: - api_list = apis.split(",") - for api in api_list: - # Validate that the API is valid - if api not in ENDPOINT_MAPPING: - raise ValueError( - f"Unknown API '{api}'. Available APIs: {list(ENDPOINT_MAPPING.keys())}" - ) - - # Check for duplicate APIs across different arguments - if self.IsRestricted(api): - raise ValueError( - f"restricted api '{api}' can not be specified in multiple config groups" - ) - - self.Insert(api, (key, value)) - - def RestrictionDict(self) -> dict[str, tuple[str, str]]: - """ - Get a copy of the restrictions dictionary. - - Returns: - dict: Copy of the restrictions mapping API names to (header_key, header_value) tuples - """ - return self._restrictions.copy() - - def Insert(self, api: str, restriction: tuple[str, str]): - """ - Add a restriction for a specific API. - - Args: - api: The API name (e.g., "inference", "model-repository") - restriction: Tuple of (header_key, header_value) for authentication - """ - self._restrictions[api] = restriction - - def IsRestricted(self, api: str) -> bool: - """ - Check if a specific API is restricted. - - Args: - api: The API name to check - - Returns: - bool: True if the API is restricted, False otherwise - """ - return api in self._restrictions - - -class APIRestrictionMiddleware(BaseHTTPMiddleware): - """ - Middleware to restrict API endpoint access based on allowed APIs configuration. - - This middleware intercepts HTTP requests and checks if they match any restricted - API endpoints. If a request matches a restricted endpoint, it validates the - authentication headers before allowing the request to proceed. - - Similar to Triton Server's endpoint access control feature. - """ - - def __init__(self, app, restricted_apis: RestrictedFeatures): - """ - Initialize the API restriction middleware. - - Args: - app: The FastAPI application instance - restricted_apis: RestrictedFeatures instance containing the restriction configuration - """ - super().__init__(app) - self.restricted_apis = restricted_apis - - def _get_auth_header(self, request: Request) -> tuple[str, str] | None: - request_method = request.method - request_path = request.url.path - - # Check each restricted API to see if the request matches - for ( - restricted_api, - auth_spec, - ) in self.restricted_apis.RestrictionDict().items(): - # Check each endpoint in the API - for restricted_endpoint in ENDPOINT_MAPPING[restricted_api]: - restricted_method, restricted_path = restricted_endpoint.split(" ") - - # Match both HTTP method and path prefix - if request_method == restricted_method and request_path.startswith( - restricted_path - ): - return auth_spec - return None - - async def dispatch(self, request: Request, call_next): - """ - Main middleware dispatch method that processes each incoming request. - - Args: - request: The incoming HTTP request - call_next: The next middleware/handler in the chain - - Returns: - Response: Either the next handler's response or a 401 authentication error - """ - # Check if the request matches any restricted patterns - auth_header = self._get_auth_header(request) - - # If request not restricted, proceed with the request - if not auth_header: - return await call_next(request) - - # Check authentication for the matching restricted endpoint - auth_result = self._check_authentication(request, auth_header) - if auth_result["valid"]: - # Authentication passed, allow request to proceed - return await call_next(request) - else: - # Authentication failed, return 401 error - return JSONResponse( - status_code=StatusCode.AUTHORIZATION_ERROR, - content={ - "error": { - "message": auth_result["message"], - "type": "authentication_error", - "code": "invalid_auth", - } - }, - ) - - def _check_authentication(self, request: Request, auth_header: tuple[str, str]): - """ - Check if the request contains valid authentication headers. - - Args: - request: The incoming HTTP request - auth_header: Tuple of (expected_header_key, expected_header_value) - - Returns: - dict: {"valid": bool, "message": str} - Authentication result and error message if invalid - """ - expected_key, expected_value = auth_header - - # Get the actual header value from the request - actual_value = request.headers.get(expected_key) - - # Validate the header value matches the expected value - if not actual_value or actual_value != expected_value: - return { - "valid": False, - "message": f"This API is restricted, expecting header '{expected_key}' with valid value", - } - - return {"valid": True} diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/request_size.py b/python/openai/openai_frontend/frontend/fastapi/middleware/request_size.py deleted file mode 100644 index 333a7968f4..0000000000 --- a/python/openai/openai_frontend/frontend/fastapi/middleware/request_size.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from fastapi.responses import JSONResponse -from starlette.types import ASGIApp, Message, Receive, Scope, Send -from utils.utils import StatusCode, validate_positive_int - - -async def _disconnect_receive() -> Message: - return {"type": "http.disconnect"} - - -class RequestSizeLimitMiddleware: - """ - Reject HTTP requests whose body exceeds ``http_max_input_size`` bytes. - First validation rejects on the Content-Length header before any body bytes are - read. Second validation streams the body chunks, counting bytes as they arrive, - and rejects as soon as the running total crosses the limit. Driving - receive() from the middleware protects every endpoint, including - handlers that never read the body. The buffered body is released the - moment the application consumes it, so the middleware contributes no - sustained memory overhead. - """ - - def __init__(self, app: ASGIApp, http_max_input_size: int) -> None: - self.app = app - self.http_max_input_size = validate_positive_int(http_max_input_size) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - # Stage 1: reject on Content-Length before reading any body bytes. - for name, value in scope["headers"]: - if name != b"content-length": - continue - try: - content_length = int(value) - except ValueError: - await self._send_error( - scope, - send, - StatusCode.CLIENT_ERROR, - "invalid_content_length", - "Invalid Content-Length header: not an integer.", - ) - return - if content_length < 0: - await self._send_error( - scope, - send, - StatusCode.CLIENT_ERROR, - "invalid_content_length", - "Invalid Content-Length header: must be non-negative.", - ) - return - if content_length > self.http_max_input_size: - await self._send_error( - scope, - send, - StatusCode.CONTENT_TOO_LARGE, - "content_too_large", - self._oversized_request_message( - content_length, self.http_max_input_size - ), - ) - return - break - - # Stage 2: count chunks as they arrive, reject if total exceeds limit. - body_chunks: list[bytes] = [] - total = 0 - while True: - message = await receive() - if message["type"] != "http.request": - return - chunk = message.get("body", b"") - total += len(chunk) - if total > self.http_max_input_size: - await self._send_error( - scope, - send, - StatusCode.CONTENT_TOO_LARGE, - "content_too_large", - self._oversized_request_message(total, self.http_max_input_size), - ) - return - body_chunks.append(chunk) - if not message.get("more_body", False): - break - - # Assemble the buffered body and replay it to the app. - body_message: Message = { - "type": "http.request", - "body": b"".join(body_chunks), - "more_body": False, - } - del body_chunks - - async def replay_receive() -> Message: - nonlocal body_message - if body_message is not None: - # Drop the reference on hand-off so the body is freed while - # the app processes it, instead of being held by this closure. - message, body_message = body_message, None - return message - # Body already delivered — delegate to the original receive() so - # streaming responses can wait for the real client disconnect. - return await receive() - - await self.app(scope, replay_receive, send) - - @staticmethod - def _oversized_request_message(actual_bytes: int, max_bytes: int) -> str: - return ( - f"Request size of {actual_bytes} bytes exceeds the maximum allowed " - f"input size of {max_bytes} bytes. " - f"Use --http-max-input-size to increase the limit." - ) - - async def _send_error( - self, - scope: Scope, - send: Send, - status_code: int, - code: str, - message: str, - ) -> None: - response = JSONResponse( - status_code=status_code, - content={ - "error": { - "message": message, - "type": "invalid_request_error", - "code": code, - } - }, - ) - await response(scope, _disconnect_receive, send) diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/chat.py b/python/openai/openai_frontend/frontend/fastapi/routers/chat.py index 49d1c5f23d..0f72047a5e 100644 --- a/python/openai/openai_frontend/frontend/fastapi/routers/chat.py +++ b/python/openai/openai_frontend/frontend/fastapi/routers/chat.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,12 +24,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import traceback - from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse from schemas.openai import CreateChatCompletionRequest, CreateChatCompletionResponse -from utils.utils import ClientError, ServerError, StatusCode router = APIRouter() @@ -45,20 +42,12 @@ async def create_chat_completion( Creates a chat completion for the provided messages and parameters. """ if not raw_request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) + raise HTTPException(status_code=500, detail="No attached inference engine") try: response = await raw_request.app.engine.chat(request) if request.stream: return StreamingResponse(response, media_type="text/event-stream") return response - except ClientError as e: - raise HTTPException(status_code=StatusCode.CLIENT_ERROR, detail=f"{e}") - except ServerError as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") except Exception as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") + raise HTTPException(status_code=400, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/completions.py b/python/openai/openai_frontend/frontend/fastapi/routers/completions.py index 642bc117d0..ade89a47cc 100644 --- a/python/openai/openai_frontend/frontend/fastapi/routers/completions.py +++ b/python/openai/openai_frontend/frontend/fastapi/routers/completions.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,12 +24,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import traceback - from fastapi import APIRouter, HTTPException, Request from fastapi.responses import StreamingResponse from schemas.openai import CreateCompletionRequest, CreateCompletionResponse -from utils.utils import ClientError, ServerError, StatusCode router = APIRouter() @@ -44,20 +41,12 @@ async def create_completion( Creates a completion for the provided prompt and parameters. """ if not raw_request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) + raise HTTPException(status_code=500, detail="No attached inference engine") try: response = await raw_request.app.engine.completion(request) if request.stream: return StreamingResponse(response, media_type="text/event-stream") return response - except ClientError as e: - raise HTTPException(status_code=StatusCode.CLIENT_ERROR, detail=f"{e}") - except ServerError as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") except Exception as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") + raise HTTPException(status_code=400, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py deleted file mode 100644 index eb2ea5d9da..0000000000 --- a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import traceback - -from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import StreamingResponse -from schemas.openai import CreateEmbeddingRequest, CreateEmbeddingResponse -from utils.utils import ClientError, ServerError, StatusCode - -router = APIRouter() - - -@router.post( - "/v1/embeddings", response_model=CreateEmbeddingResponse, tags=["Embeddings"] -) -async def create_embedding( - request: CreateEmbeddingRequest, raw_request: Request -) -> CreateEmbeddingResponse | StreamingResponse: - """ - Creates embedding for the provided input text. - """ - if not raw_request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) - - try: - response = await raw_request.app.engine.embedding(request) - return response - except ClientError as e: - raise HTTPException(status_code=StatusCode.CLIENT_ERROR, detail=f"{e}") - except ServerError as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") - except Exception as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/model_management.py b/python/openai/openai_frontend/frontend/fastapi/routers/model_management.py deleted file mode 100644 index 2ddecb8883..0000000000 --- a/python/openai/openai_frontend/frontend/fastapi/routers/model_management.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import traceback - -from fastapi import APIRouter, HTTPException, Request -from schemas.openai import Model -from utils.utils import ClientError, ServerError, StatusCode - -router = APIRouter() - - -@router.post( - "/v1/models/{model_name}/load", - response_model=Model, - tags=["Model Management"], -) -async def load_model(model_name: str, raw_request: Request) -> Model: - """ - Loads a model by name. Only available in EXPLICIT model control mode. - Blocks until the model is fully loaded and ready. - """ - if not raw_request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, - detail="No attached inference engine", - ) - - try: - return await raw_request.app.engine.load_model(model_name) - except ClientError as e: - raise HTTPException(status_code=StatusCode.CLIENT_ERROR, detail=f"{e}") - except ServerError as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") - except Exception as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") - - -@router.post( - "/v1/models/{model_name}/unload", - tags=["Model Management"], -) -async def unload_model(model_name: str, raw_request: Request) -> dict: - """ - Unloads a model by name. Only available in EXPLICIT model control mode. - Blocks until the model is fully unloaded. In-flight requests are allowed - to complete before the model is removed. - """ - if not raw_request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, - detail="No attached inference engine", - ) - - try: - await raw_request.app.engine.unload_model(model_name) - return {"status": "success", "model": model_name} - except ClientError as e: - raise HTTPException(status_code=StatusCode.CLIENT_ERROR, detail=f"{e}") - except ServerError as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") - except Exception as e: - print(traceback.format_exc()) - raise HTTPException(status_code=StatusCode.SERVER_ERROR, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/models.py b/python/openai/openai_frontend/frontend/fastapi/routers/models.py index 871e300276..ac2fa7fdc0 100644 --- a/python/openai/openai_frontend/frontend/fastapi/routers/models.py +++ b/python/openai/openai_frontend/frontend/fastapi/routers/models.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,7 +28,6 @@ from fastapi import APIRouter, HTTPException, Request from schemas.openai import ListModelsResponse, Model, ObjectType -from utils.utils import StatusCode router = APIRouter() @@ -36,28 +35,24 @@ @router.get("/v1/models", response_model=ListModelsResponse, tags=["Models"]) -async def list_models(request: Request) -> ListModelsResponse: +def list_models(request: Request) -> ListModelsResponse: """ Lists the currently available models, and provides basic information about each one such as the owner and availability. """ if not request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) + raise HTTPException(status_code=500, detail="No attached inference engine") models: List[Model] = request.app.engine.models() return ListModelsResponse(object=ObjectType.list, data=models) @router.get("/v1/models/{model_name}", response_model=Model, tags=["Models"]) -async def retrieve_model(request: Request, model_name: str) -> Model: +def retrieve_model(request: Request, model_name: str) -> Model: """ Retrieves a model instance, providing basic information about the model such as the owner and permissioning. """ if not request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) + raise HTTPException(status_code=500, detail="No attached inference engine") # TODO: Return model directly from engine instead of searching models models: List[Model] = request.app.engine.models() @@ -65,6 +60,4 @@ async def retrieve_model(request: Request, model_name: str) -> Model: if model.id == model_name: return model - raise HTTPException( - status_code=StatusCode.NOT_FOUND, detail=f"Unknown model: {model_name}" - ) + raise HTTPException(status_code=404, detail=f"Unknown model: {model_name}") diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/observability.py b/python/openai/openai_frontend/frontend/fastapi/routers/observability.py index ca881c632e..b8040f56b7 100644 --- a/python/openai/openai_frontend/frontend/fastapi/routers/observability.py +++ b/python/openai/openai_frontend/frontend/fastapi/routers/observability.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,7 +26,6 @@ from fastapi import APIRouter, HTTPException, Request from fastapi.responses import PlainTextResponse, Response -from utils.utils import StatusCode router = APIRouter() @@ -39,14 +38,12 @@ def metrics(request: Request) -> PlainTextResponse: @router.get("/health/ready", tags=["Utilities"]) def ready(request: Request) -> Response: if not request.app.engine: - raise HTTPException( - status_code=StatusCode.SERVER_ERROR, detail="No attached inference engine" - ) + raise HTTPException(status_code=500, detail="No attached inference engine") if not request.app.engine.ready(): raise HTTPException( - status_code=StatusCode.CLIENT_ERROR, + status_code=400, detail="Attached inference engine is not ready for inference requests.", ) - return Response(status_code=StatusCode.SUCCESS) + return Response(status_code=200) diff --git a/python/openai/openai_frontend/frontend/fastapi_frontend.py b/python/openai/openai_frontend/frontend/fastapi_frontend.py index 752befd8bc..adee4cbab3 100644 --- a/python/openai/openai_frontend/frontend/fastapi_frontend.py +++ b/python/openai/openai_frontend/frontend/fastapi_frontend.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,21 +30,8 @@ from engine.triton_engine import TritonLLMEngine from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from frontend.fastapi.middleware.api_restriction import ( - APIRestrictionMiddleware, - RestrictedFeatures, -) -from frontend.fastapi.middleware.request_size import RequestSizeLimitMiddleware -from frontend.fastapi.routers import ( - chat, - completions, - embeddings, - model_management, - models, - observability, -) +from frontend.fastapi.routers import chat, completions, models, observability from frontend.frontend import OpenAIFrontend -from utils.utils import HTTP_DEFAULT_MAX_INPUT_SIZE class FastApiFrontend(OpenAIFrontend): @@ -54,19 +41,10 @@ def __init__( host: str = "localhost", port: int = 8000, log_level: str = "info", - restricted_apis: list = None, - http_max_input_size: int = HTTP_DEFAULT_MAX_INPUT_SIZE, ): self.host: str = host self.port: int = port self.log_level: str = log_level - self.http_max_input_size: int = http_max_input_size - if restricted_apis: - self.restricted_apis: RestrictedFeatures = RestrictedFeatures( - restricted_apis - ) - else: - self.restricted_apis: RestrictedFeatures = None self.stopped: bool = False self.app = self._create_app() @@ -106,16 +84,11 @@ def _create_app(self): app.include_router(observability.router) app.include_router(models.router) - app.include_router(model_management.router) app.include_router(completions.router) app.include_router(chat.router) - app.include_router(embeddings.router) # NOTE: For debugging purposes, should generally be restricted or removed self._add_cors_middleware(app) - if self.restricted_apis != None: - self._add_api_restriction_middleware(app) - self._add_request_size_limit_middleware(app) return app @@ -134,18 +107,3 @@ def _add_cors_middleware(self, app: FastAPI): allow_methods=["*"], allow_headers=["*"], ) - - def _add_api_restriction_middleware(self, app: FastAPI): - app.add_middleware( - APIRestrictionMiddleware, restricted_apis=self.restricted_apis - ) - print( - f"[INFO] API restrictions enabled. Restricted API endpoints: {self.restricted_apis.RestrictionDict()}" - ) - - def _add_request_size_limit_middleware(self, app: FastAPI): - app.add_middleware( - RequestSizeLimitMiddleware, - http_max_input_size=self.http_max_input_size, - ) - print(f"[INFO] HTTP request size limit set to {self.http_max_input_size} bytes") diff --git a/python/openai/openai_frontend/main.py b/python/openai/openai_frontend/main.py index cf9228b9fc..5a4e2368a0 100755 --- a/python/openai/openai_frontend/main.py +++ b/python/openai/openai_frontend/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,13 +28,11 @@ import argparse import signal -import sys from functools import partial import tritonserver from engine.triton_engine import TritonLLMEngine from frontend.fastapi_frontend import FastApiFrontend -from utils.utils import HTTP_DEFAULT_MAX_INPUT_SIZE, validate_positive_int def signal_handler( @@ -110,12 +108,6 @@ def parse_args(): choices=["vllm", "tensorrtllm"], help="Manual override of Triton backend request format (inputs/output names) to use for inference", ) - triton_group.add_argument( - "--lora-separator", - type=str, - default=None, - help="LoRA name selection may be appended to the model name following this separator if the separator is provided", - ) triton_group.add_argument( "--tritonserver-log-verbose-level", type=int, @@ -128,54 +120,6 @@ def parse_args(): default="0.0.0.0", help="Address/host of frontends (default: '0.0.0.0')", ) - triton_group.add_argument( - "--tool-call-parser", - type=str, - default=None, - help="Specify the parser for handling tool calling related response text. Options include: 'llama3' and 'mistral'.", - ) - # Allows the user to try a different chat template to craft better prompts and receive more targeted tool-calling responses from the model. - # Some Mistral models have a separate chat template file, in addition to the tokenizer_config.json, - # such as mistralai/Mistral-Small-3.1-24B-Instruct-2503. - # This can serve as a workaround for those models. - triton_group.add_argument( - "--chat-template", - type=str, - default=None, - help="The path to the custom Jinja chat template file. This is useful if you'd like to use a different chat template than the one provided by the model.", - ) - - triton_group.add_argument( - "--default-max-tokens", - type=int, - default=16, - help="The default maximum number of tokens to generate if not specified in the request. The default is 16.", - ) - triton_group.add_argument( - "--model-control-mode", - type=str, - default="none", - choices=["none", "explicit"], - help="Specify the mode for model management. Options are 'none', and 'explicit'. " - "The default is 'none'. For 'none', the server will load all models in the model " - "repository at startup and will not make any changes to the loaded " - "models after that. For 'explicit', model load and unload are initiated by using the " - "model control APIs, and only models specified with --load-model will " - "be loaded at startup.", - ) - triton_group.add_argument( - "--load-model", - type=str, - action="append", - default=None, - help="Name of the model to be loaded on server startup. It may be specified " - "multiple times to add multiple models. To load ALL models at startup, " - "specify '*' as the model name with --load-model=* as the ONLY " - "--load-model argument, this does not imply any pattern matching. " - "Specifying --load-model=* in conjunction with another --load-model " - "argument will result in error. Note that this option will only take " - "effect if --model-control-mode is set to 'explicit'.", - ) # OpenAI-Compatible Frontend (FastAPI) openai_group = parser.add_argument_group("Triton OpenAI-Compatible Frontend") @@ -189,23 +133,6 @@ def parse_args(): choices=["debug", "info", "warning", "error", "critical", "trace"], help="log level for uvicorn", ) - openai_group.add_argument( - "--openai-restricted-api", - type=str, - default=None, - nargs=3, - metavar=("APIs", "Restricted Key", "Restricted Value"), - action="append", - help="Restrict access to specific OpenAI API endpoints. Format: ',,... ' (e.g., 'inference,model-repository admin-key admin-value'). If not specified, all endpoints are allowed.", - ) - openai_group.add_argument( - "--http-max-input-size", - type=validate_positive_int, - default=HTTP_DEFAULT_MAX_INPUT_SIZE, - help=f"Maximum allowed HTTP request input size in bytes for the OpenAI " - f"frontend (default: {HTTP_DEFAULT_MAX_INPUT_SIZE}, i.e. 64 MiB). " - "Requests exceeding this limit will be rejected.", - ) # KServe Predict v2 Frontend kserve_group = parser.add_argument_group("Triton KServe Frontend") @@ -234,25 +161,8 @@ def main(): args = parse_args() # Initialize a Triton Inference Server pointing at LLM models - model_control_mode = ( - tritonserver.ModelControlMode.EXPLICIT - if args.model_control_mode == "explicit" - else tritonserver.ModelControlMode.NONE - ) - - load_models = args.load_model or [] - if load_models and model_control_mode != tritonserver.ModelControlMode.EXPLICIT: - print( - "Error: Use of '--load-model' requires setting " - "'--model-control-mode=explicit' as well.", - file=sys.stderr, - ) - sys.exit(1) - server: tritonserver.Server = tritonserver.Server( model_repository=args.model_repository, - model_control_mode=model_control_mode, - startup_models=load_models, log_verbose=args.tritonserver_log_verbose_level, log_info=True, log_warn=True, @@ -261,31 +171,16 @@ def main(): # Wrap Triton Inference Server in an interface-conforming "LLMEngine" engine: TritonLLMEngine = TritonLLMEngine( - server=server, - tokenizer=args.tokenizer, - backend=args.backend, - lora_separator=args.lora_separator, - tool_call_parser=args.tool_call_parser, - chat_template=args.chat_template, - default_max_tokens=args.default_max_tokens, + server=server, tokenizer=args.tokenizer, backend=args.backend ) # Attach TritonLLMEngine as the backbone for inference and model management - try: - openai_frontend: FastApiFrontend = FastApiFrontend( - engine=engine, - host=args.host, - port=args.openai_port, - log_level=args.uvicorn_log_level, - restricted_apis=args.openai_restricted_api, - http_max_input_size=args.http_max_input_size, - ) - except ValueError as e: - print( - f"[ERROR] Failed to initialize FastAPI frontend: {e}", - file=sys.stderr, - ) - sys.exit(1) + openai_frontend: FastApiFrontend = FastApiFrontend( + engine=engine, + host=args.host, + port=args.openai_port, + log_level=args.uvicorn_log_level, + ) # Optionally expose Triton KServe HTTP/GRPC Frontends kserve_http, kserve_grpc = None, None diff --git a/python/openai/openai_frontend/schemas/openai.py b/python/openai/openai_frontend/schemas/openai.py index 57fb7b1017..8b5545d910 100644 --- a/python/openai/openai_frontend/schemas/openai.py +++ b/python/openai/openai_frontend/schemas/openai.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,7 +31,7 @@ from __future__ import annotations from enum import Enum -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Optional, Union from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel, confloat, conint @@ -103,7 +103,7 @@ class CreateCompletionRequest(BaseModel): description="Include the log probabilities on the `logprobs` most likely output tokens, as well the chosen tokens. For example, if `logprobs` is 5, the API will return a list of the 5 most likely tokens. The API will always return the `logprob` of the sampled token, so there may be up to `logprobs+1` elements in the response.\n\nThe maximum value for `logprobs` is 5.\n", ) max_tokens: Optional[conint(ge=0)] = Field( - None, + 16, description="The maximum number of [tokens](/tokenizer) that can be generated in the completion.\n\nThe token count of your prompt plus `max_tokens` cannot exceed the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", examples=[16], ) @@ -133,10 +133,6 @@ class CreateCompletionRequest(BaseModel): False, description="Whether to stream back partial progress. If set, tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n", ) - stream_options: Optional[StreamOptions] = Field( - None, - description="Options for streaming responses. Only use when `stream` is set to `true`.", - ) suffix: Optional[str] = Field( None, description="The suffix that comes after a completion of inserted text.\n\nThis parameter is only supported for `gpt-3.5-turbo-instruct`.\n", @@ -324,6 +320,10 @@ class ChatCompletionFunctionCallOption(BaseModel): name: str = Field(..., description="The name of the function to call.") +class Type2(Enum): + function = "function" + + class FunctionObject(BaseModel): description: Optional[str] = Field( None, @@ -347,7 +347,7 @@ class Function(BaseModel): class ChatCompletionNamedToolChoice(BaseModel): - type: str = Field( + type: Type2 = Field( ..., description="The type of the tool. Currently, only `function` is supported.", ) @@ -364,7 +364,7 @@ class Function1(BaseModel): class ChatCompletionMessageToolCall(BaseModel): id: str = Field(..., description="The ID of the tool call.") - type: str = Field( + type: Type2 = Field( ..., description="The type of the tool. Currently, only `function` is supported.", ) @@ -382,7 +382,7 @@ class Function2(BaseModel): class ChatCompletionMessageToolCallChunk(BaseModel): index: int id: Optional[str] = Field(None, description="The ID of the tool call.") - type: Optional[str] = Field( + type: Optional[Type2] = Field( None, description="The type of the tool. Currently, only `function` is supported.", ) @@ -471,13 +471,6 @@ class ResponseFormat(BaseModel): ) -class StreamOptions(BaseModel): - include_usage: Optional[bool] = Field( - False, - description="If enabled, an additional chunk is sent before the `data: [DONE]` message. That chunk’s `usage` field reports the total token usage for the request and its `choices` array is always empty. All other chunks include a `usage` field with a null value.", - ) - - class FunctionCall3(Enum): none = "none" auto = "auto" @@ -530,16 +523,24 @@ class ChatCompletionTokenLogprob(BaseModel): ) -class ChatCompletionLogprobs(BaseModel): +class Logprobs2(BaseModel): content: List[ChatCompletionTokenLogprob] = Field( ..., description="A list of message content tokens with log probability information.", ) +class ChatCompletionFinishReason(Enum): + stop = "stop" + length = "length" + tool_calls = "tool_calls" + content_filter = "content_filter" + function_call = "function_call" + + class ChatCompletionStreamingResponseChoice(BaseModel): delta: ChatCompletionStreamResponseDelta - logprobs: Optional[ChatCompletionLogprobs] = Field( + logprobs: Optional[Logprobs2] = Field( None, description="Log probability information for the choice." ) finish_reason: ChatCompletionFinishReason | None = Field( @@ -576,7 +577,6 @@ class CreateChatCompletionStreamResponse(BaseModel): object: Object4 = Field( ..., description="The object type, which is always `chat.completion.chunk`." ) - usage: Optional[CompletionUsage] = None class CreateChatCompletionImageResponse(BaseModel): @@ -601,7 +601,10 @@ class Model(BaseModel): owned_by: str = Field(..., description="The organization that owns the model.") -class BaseUsage(BaseModel): +class CompletionUsage(BaseModel): + completion_tokens: int = Field( + ..., description="Number of tokens in the generated completion." + ) prompt_tokens: int = Field(..., description="Number of tokens in the prompt.") total_tokens: int = Field( ..., @@ -609,16 +612,6 @@ class BaseUsage(BaseModel): ) -class EmbeddingUsage(BaseUsage): - pass - - -class CompletionUsage(BaseUsage): - completion_tokens: int = Field( - ..., description="Number of tokens in the generated completion." - ) - - class Event(Enum): error = "error" @@ -690,7 +683,7 @@ class ChatCompletionRequestUserMessage(BaseModel): class ChatCompletionTool(BaseModel): - type: str = Field( + type: Type2 = Field( ..., description="The type of the tool. Currently, only `function` is supported.", ) @@ -730,7 +723,7 @@ class ChatCompletionChoice(BaseModel): ..., description="The index of the choice in the list of choices." ) message: ChatCompletionResponseMessage - logprobs: ChatCompletionLogprobs | None = Field( + logprobs: Logprobs2 | None = Field( ..., description="Log probability information for the choice." ) @@ -861,14 +854,10 @@ class CreateChatCompletionRequest(BaseModel): None, description="An integer between 0 and 20 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to `true` if this parameter is used.", ) - max_completion_tokens: Optional[conint(ge=0)] = Field( - None, - description="The maximum number of [tokens](/tokenizer) that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", - ) - # TODO: Remove support for max_tokens field in the future: https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_completion_tokens + # TODO: Consider new max_completion_tokens field in the future: https://platform.openai.com/docs/api-reference/chat/create#chat-create-max_completion_tokens max_tokens: Optional[conint(ge=0)] = Field( - None, - description="DEPRECATED: Use `max_completion_tokens` instead. The maximum number of [tokens](/tokenizer) that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", + 16, + description="The maximum number of [tokens](/tokenizer) that can be generated in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken) for counting tokens.\n", ) # TODO: Extension, flesh out description and defaults min_tokens: Optional[conint(ge=0)] = Field( @@ -886,7 +875,7 @@ class CreateChatCompletionRequest(BaseModel): ) response_format: Optional[ResponseFormat] = Field( None, - description='An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`.\n\nSetting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON.\n\n**Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_completion_tokens` or the conversation exceeded the max context length.\n', + description='An object specifying the format that the model must output. Compatible with [GPT-4 Turbo](/docs/models/gpt-4-and-gpt-4-turbo) and all GPT-3.5 Turbo models newer than `gpt-3.5-turbo-1106`.\n\nSetting to `{ "type": "json_object" }` enables JSON mode, which guarantees the message the model generates is valid JSON.\n\n**Important:** when using JSON mode, you **must** also instruct the model to produce JSON yourself via a system or user message. Without this, the model may generate an unending stream of whitespace until the generation reaches the token limit, resulting in a long-running and seemingly "stuck" request. Also note that the message content may be partially cut off if `finish_reason="length"`, which indicates the generation exceeded `max_tokens` or the conversation exceeded the max context length.\n', ) seed: Optional[conint(ge=-9223372036854775808, le=9223372036854775807)] = Field( None, @@ -900,10 +889,6 @@ class CreateChatCompletionRequest(BaseModel): False, description="If set, partial message deltas will be sent, like in ChatGPT. Tokens will be sent as data-only [server-sent events](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_format) as they become available, with the stream terminated by a `data: [DONE]` message. [Example Python code](https://cookbook.openai.com/examples/how_to_stream_completions).\n", ) - stream_options: Optional[StreamOptions] = Field( - None, - description="Options for streaming responses. Only use when `stream` is set to `true`.", - ) temperature: Optional[confloat(ge=0.0, le=2.0)] = Field( 0.7, description="What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n", @@ -947,69 +932,3 @@ class ObjectType: text_completion = Object1.text_completion chat_completion_chunk = Object4.chat_completion_chunk chat_completion = Object2.chat_completion - - -class EmbeddingObject(BaseModel): - model_config: ConfigDict = ConfigDict(extra="forbid") - - object: Literal["embedding"] = Field( - description="The object type, which is always 'embedding'.", - ) - embedding: Union[List[float], str] = Field( - ..., - description="The embedding vector, which is a list of floats or a base64-encoded string.", - ) - index: int = Field( - ..., - description="The index of the embedding in the list of embeddings.", - ) - - -class CreateEmbeddingRequest(BaseModel): - # Explicitly return errors for unknown fields. - model_config: ConfigDict = ConfigDict(extra="forbid") - - input: Union[str, List[int]] = Field( - ..., - description="Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.", - min_length=1, - examples=["The food was delicious and the waiter..."], - ) - model: Union[str, Model2] = Field( - ..., - description="ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API.", - examples=["text-embedding-ada-002"], - ) - dimensions: Optional[int] = Field( - None, - description="The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.", - ) - encoding_format: Optional[Literal["float", "base64"]] = Field( - "float", - description="The format to return the embeddings in.", - ) - user: Optional[str] = Field( - None, - description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n", - examples=["user-1234"], - ) - - -class CreateEmbeddingResponse(BaseModel): - model_config: ConfigDict = ConfigDict(extra="forbid") - - object: Literal["list"] = Field( - description="The object type, which is always 'list'.", - ) - data: List[EmbeddingObject] = Field( - ..., - description="The list of embeddings.", - ) - model: Union[str, Model2] = Field( - ..., - description="The model used to generate the embeddings.", - ) - usage: Optional[EmbeddingUsage] = Field( - ..., - description="The usage for the request.", - ) diff --git a/python/openai/openai_frontend/utils/utils.py b/python/openai/openai_frontend/utils/utils.py deleted file mode 100644 index 86b4d921e4..0000000000 --- a/python/openai/openai_frontend/utils/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from enum import IntEnum - -# Default value for the --http-max-input-size CLI flag (64 MiB). -# Same as HTTP_DEFAULT_MAX_INPUT_SIZE in src/common.h. -HTTP_DEFAULT_MAX_INPUT_SIZE: int = 1 << 26 - - -class ServerError(Exception): - """Exception raised for server errors.""" - - pass - - -class ClientError(Exception): - """Exception raised for client errors.""" - - pass - - -class StatusCode(IntEnum): - SUCCESS = 200 - CLIENT_ERROR = 400 - AUTHORIZATION_ERROR = 401 - NOT_FOUND = 404 - CONTENT_TOO_LARGE = 413 - SERVER_ERROR = 500 - - -def validate_positive_int(value: object) -> int: - try: - ivalue = int(value) - except (TypeError, ValueError): - raise ValueError(f"value is not an integer, got {value!r}") - if ivalue <= 0: - raise ValueError(f"value must be greater than 0, got {value!r}") - return ivalue diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index 4b4f631f9b..ac84944631 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,16 +25,11 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # FastAPI Application -fastapi==0.121.2 +fastapi==0.115.6 # Fix httpx version to avoid bug in openai library: # https://community.openai.com/t/error-with-openai-1-56-0-client-init-got-an-unexpected-keyword-argument-proxies/1040332/3 httpx==0.27.2 -openai==1.107.3 -partial-json-parser # used for parsing partial JSON outputs - -# FIXME [TRI-641]: The latest stable version of scipy is 1.17.0 which caused segfault during tests. See TRI-620. -scipy==1.16.3 -# Minimum starlette version needed to address CVE(s): +openai==1.60.0 +# Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw -# https://github.com/advisories/GHSA-7f5h-v6xp-fcq8 -starlette>=0.49.1 +starlette>=0.40.0 diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index a1195000ed..9ea9a5634e 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,25 +32,14 @@ from tests.utils import OpenAIServer, setup_fastapi_app, setup_server -def pytest_configure(config): - """Register custom markers.""" - config.addinivalue_line( - "markers", "openai: mark test to run with OpenAI server (subprocess)" - ) - config.addinivalue_line("markers", "asyncio: mark test as an asyncio test") - - ### TEST ENVIRONMENT SETUP ### -def infer_test_environment(tool_call_parser): +def infer_test_environment(): # Infer the test environment for simplicity in local dev/testing. try: import vllm as _ backend = "vllm" - if tool_call_parser == "mistral": - model = "mistral-nemo-instruct-2407" - else: - model = "llama-3.1-8b-instruct" + model = "llama-3.1-8b-instruct" return backend, model except ImportError: print("No vllm installation found.") @@ -67,88 +56,41 @@ def infer_test_environment(tool_call_parser): raise Exception("Unknown test environment") -def infer_test_model_repository(backend, tool_call_parser): - if tool_call_parser == "mistral": - model_repository = str(Path(__file__).parent / f"{backend}_mistral_models") - else: - model_repository = str(Path(__file__).parent / f"{backend}_models") +def infer_test_model_repository(backend): + model_repository = str(Path(__file__).parent / f"{backend}_models") return model_repository -### FIXTURES - Refactored from global variables ### - - -@pytest.fixture(scope="session") -def tool_call_parser(): - return os.environ.get("TEST_TOOL_CALL_PARSER", "llama3") - - -@pytest.fixture(scope="session") -def backend(tool_call_parser): - env_backend = os.environ.get("TEST_BACKEND") - env_model = os.environ.get("TEST_MODEL") - - if not env_backend or not env_model: - inferred_backend, _ = infer_test_environment(tool_call_parser) - return inferred_backend - return env_backend - - -@pytest.fixture(scope="session") -def model(tool_call_parser): - env_model = os.environ.get("TEST_MODEL") - - if not env_model: - _, inferred_model = infer_test_environment(tool_call_parser) - return inferred_model - return env_model - - -@pytest.fixture(scope="session") -def model_repository(backend, tool_call_parser): - env_repo = os.environ.get("TEST_MODEL_REPOSITORY") - - if env_repo: - return env_repo - return infer_test_model_repository(backend, tool_call_parser) - - -@pytest.fixture(scope="session") -def tokenizer_model(): - return os.environ.get("TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct") - - -@pytest.fixture(scope="session") -def prompt(): - return "What is machine learning?" - +# TODO: Refactor away from global variables +TEST_MODEL = os.environ.get("TEST_MODEL") +TEST_BACKEND = os.environ.get("TEST_BACKEND") +TEST_MODEL_REPOSITORY = os.environ.get("TEST_MODEL_REPOSITORY") -@pytest.fixture(scope="session") -def messages(prompt): - return [{"role": "user", "content": prompt}] +TEST_TOKENIZER = os.environ.get( + "TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct" +) +TEST_PROMPT = "What is machine learning?" +TEST_MESSAGES = [{"role": "user", "content": TEST_PROMPT}] +if not TEST_BACKEND or not TEST_MODEL: + TEST_BACKEND, TEST_MODEL = infer_test_environment() -@pytest.fixture(scope="session") -def input(prompt): - return prompt +if not TEST_MODEL_REPOSITORY: + TEST_MODEL_REPOSITORY = infer_test_model_repository(TEST_BACKEND) # NOTE: OpenAI client requires actual server running, and won't work # with the FastAPI TestClient. Run the server at module scope to run # only once for all the tests below. @pytest.fixture(scope="module") -def server( - model_repository: str, tokenizer_model: str, backend: str, tool_call_parser: str -): +def server(): args = [ "--model-repository", - model_repository, + TEST_MODEL_REPOSITORY, "--tokenizer", - tokenizer_model, + TEST_TOKENIZER, "--backend", - backend, - "--tool-call-parser", - tool_call_parser, + TEST_BACKEND, ] # TODO: Incorporate kserve frontend binding smoke tests to catch any # breakage with default values or slight cli arg variations @@ -164,36 +106,42 @@ def server( # with arbitrary clients - you must use the TestClient returned to interact with # the "server" when "starting the server" via TestClient. @pytest.fixture(scope="class") -def fastapi_client_class_scope( - model_repository: str, tokenizer_model: str, backend: str -): - server = setup_server(model_repository=model_repository) - app = setup_fastapi_app(tokenizer=tokenizer_model, server=server, backend=backend) +def fastapi_client_class_scope(): + server = setup_server(model_repository=TEST_MODEL_REPOSITORY) + app = setup_fastapi_app( + tokenizer=TEST_TOKENIZER, server=server, backend=TEST_BACKEND + ) with TestClient(app) as test_client: yield test_client server.stop() -# FIXME: In TRTLLM tests, the in-process Triton server for the FastAPI app -# does not automatically release GPU memory, even after calling stop(). -# The memory is only released when the entire pytest process exits. -# -# As a result, when the OpenAI server starts another Triton server as a subprocess, -# there may not be enough GPU memory available to launch a new model instance. -# -# This is a workaround to ensure that tests using the OpenAI server run first. -# Once the OpenAI server subprocess is terminated, tests using the FastAPI app can safely run. -def pytest_collection_modifyitems(session, config, items): - def get_priority(item): - cls = item.cls - if cls: - if getattr(cls, "pytestmark", None): - for mark in cls.pytestmark: - if mark.name == "openai": - return 0 - elif mark.name == "fastapi": - return 1 - return 2 # unmarked tests last - - items.sort(key=get_priority) +@pytest.fixture(scope="module") +def model_repository(): + return TEST_MODEL_REPOSITORY + + +@pytest.fixture(scope="module") +def model(): + return TEST_MODEL + + +@pytest.fixture(scope="module") +def backend(): + return TEST_BACKEND + + +@pytest.fixture(scope="module") +def tokenizer_model(): + return TEST_TOKENIZER + + +@pytest.fixture(scope="module") +def prompt(): + return TEST_PROMPT + + +@pytest.fixture(scope="module") +def messages(): + return TEST_MESSAGES diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index eee18bf354..401601c526 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,9 +50,8 @@ def test_chat_completions_defaults(self, client, model: str, messages: List[dict message = response.json()["choices"][0]["message"] assert message["content"].strip() assert message["role"] == "assistant" - - usage = response.json().get("usage") - assert usage is not None + # "usage" currently not supported + assert not response.json()["usage"] def test_chat_completions_system_prompt(self, client, model: str): # NOTE: Currently just sanity check that there are no issues when a @@ -123,7 +122,6 @@ def test_chat_completions_user_prompt_dict(self, client, model: str): [ ("temperature", 0.7), ("max_tokens", 10), - ("max_completion_tokens", 10), ("top_p", 0.9), ("frequency_penalty", 0.5), ("presence_penalty", 0.2), @@ -154,22 +152,12 @@ def test_chat_completions_sampling_parameters( ) # FIXME: Add support and remove this check - unsupported_parameters = ["logit_bias"] + unsupported_parameters = ["logprobs", "logit_bias"] if param_key in unsupported_parameters: - assert response.status_code == 400 - assert response.json()["detail"] == "logit bias is not currently supported" - return - - # TRT-LLM backend doesn't support logprobs - if ( - param_key == "logprobs" - and param_value is True - and model == "tensorrt_llm_bls" - ): assert response.status_code == 400 assert ( - "logprobs are currently available only for the vLLM backend" - in response.json()["detail"] + response.json()["detail"] + == "logit bias and log probs not currently supported" ) return @@ -183,7 +171,6 @@ def test_chat_completions_sampling_parameters( ("temperature", 2.1), ("temperature", -0.1), ("max_tokens", -1), - ("max_completion_tokens", -1), ("top_p", 1.1), ("frequency_penalty", 3), ("frequency_penalty", -3), @@ -211,21 +198,14 @@ def test_chat_completions_invalid_sampling_parameters( assert response.status_code == 422 # Simple tests to verify max_tokens roughly behaves as expected - @pytest.mark.parametrize( - "max_tokens_key", - [ - "max_tokens", - "max_completion_tokens", - ], - ) def test_chat_completions_max_tokens( - self, client, max_tokens_key, model: str, messages: List[dict] + self, client, model: str, messages: List[dict] ): responses = [] - payload = {"model": model, "messages": messages} + payload = {"model": model, "messages": messages, "max_tokens": 1} - # Send two requests with max_tokens/max_completion_tokens = 1 to check their similarity - payload[max_tokens_key] = 1 + # Send two requests with max_tokens = 1 to check their similarity + payload["max_tokens"] = 1 responses.append( client.post( "/v1/chat/completions", @@ -238,8 +218,8 @@ def test_chat_completions_max_tokens( json=payload, ) ) - # Send one requests with larger max_tokens/max_completion_tokens to check its dis-similarity - payload[max_tokens_key] = 100 + # Send one requests with larger max_tokens to check its dis-similarity + payload["max_tokens"] = 100 responses.append( client.post( "/v1/chat/completions", @@ -264,30 +244,6 @@ def test_chat_completions_max_tokens( assert len(response1_text) == len(response2_text) == 1 assert len(response3_text) > len(response1_text) - def test_chat_completions_max_completion_tokens_precedence( - self, client, model: str, messages: List[dict] - ): - payload = { - "model": model, - "messages": messages, - "max_tokens": 50, # Higher value for max_tokens - "max_completion_tokens": 1, # Lower, expected to take precedence - } - - response = client.post( - "/v1/chat/completions", - json=payload, - ) - - print("Response:", response.json()) - assert response.status_code == 200 - - response_text_words = ( - response.json()["choices"][0]["message"]["content"].strip().split() - ) - # Check if the number of words is around max_completion_tokens - assert len(response_text_words) == 1 - @pytest.mark.parametrize( "temperature", [0.0, 1.0], @@ -303,7 +259,7 @@ def test_chat_completions_temperature_vllm( payload = { "model": model, "messages": messages, - "max_completion_tokens": 256, + "max_tokens": 256, "temperature": temperature, } @@ -364,7 +320,7 @@ def test_chat_completions_temperature_tensorrtllm( "model": model, "messages": messages, # Increase token length to allow more room for variability - "max_completion_tokens": 200, + "max_tokens": 200, "temperature": 0.0, # TRT-LLM requires certain settings of `top_k` / `top_p` to # respect changes in `temperature` @@ -419,7 +375,7 @@ def test_chat_completions_seed(self, client, model: str, messages: List[dict]): "model": model, "messages": messages, # Increase token length to allow more room for variability - "max_completion_tokens": 200, + "max_tokens": 200, "seed": 1, } payload2 = copy.deepcopy(payload1) @@ -533,156 +489,16 @@ def test_request_n_choices(self): pass @pytest.mark.skip(reason="Not Implemented Yet") - def test_request_logit_bias(self): + def test_request_logprobs(self): pass - def test_usage_response(self, client, model: str, messages: List[dict]): - response = client.post( - "/v1/chat/completions", - json={"model": model, "messages": messages}, - ) - - assert response.status_code == 200 - usage = response.json().get("usage") - assert usage is not None - assert isinstance(usage["prompt_tokens"], int) - assert isinstance(usage["completion_tokens"], int) - assert isinstance(usage["total_tokens"], int) - assert usage["prompt_tokens"] > 0 - assert usage["completion_tokens"] > 0 - assert ( - usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] - ) - - def test_chat_completions_logprobs( - self, client, backend: str, model: str, messages: List[dict] - ): - """Test logprobs parameter for chat completions.""" - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "logprobs": True, - "top_logprobs": 2, - "max_tokens": 10, - }, - ) - - # Non-vLLM backends should raise an error - if backend != "vllm": - assert response.status_code == 400 - assert ( - "logprobs are currently available only for the vLLM backend" - in response.json()["detail"] - ) - return - - assert response.status_code == 200 - response_json = response.json() - - # Check that logprobs are present in the response - choice = response_json["choices"][0] - assert "logprobs" in choice - logprobs = choice["logprobs"] - - assert logprobs is not None - assert "content" in logprobs - content = logprobs["content"] - assert isinstance(content, list) - assert len(content) > 0 - - # Validate structure of each token logprob - for token_logprob in content: - assert "token" in token_logprob - assert "logprob" in token_logprob - assert "bytes" in token_logprob - assert "top_logprobs" in token_logprob - - assert isinstance(token_logprob["token"], str) - assert isinstance(token_logprob["logprob"], (int, float)) - assert isinstance(token_logprob["bytes"], list) - assert isinstance(token_logprob["top_logprobs"], list) - - # Validate top_logprobs structure - for top_logprob in token_logprob["top_logprobs"]: - assert "token" in top_logprob - assert "logprob" in top_logprob - assert "bytes" in top_logprob - - def test_chat_completions_logprobs_false( - self, client, model: str, messages: List[dict] - ): - """Test that logprobs=False returns no logprobs.""" - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "logprobs": False, - "max_tokens": 10, - }, - ) - - assert response.status_code == 200 - response_json = response.json() - - # logprobs should be None when logprobs=False - choice = response_json["choices"][0] - assert choice.get("logprobs") is None - - @pytest.mark.parametrize("top_logprobs_value", [0, 5]) - def test_chat_completions_top_logprobs_without_logprobs( - self, - client, - model: str, - messages: List[dict], - top_logprobs_value: int, - backend: str, - ): - """Test that top_logprobs without logprobs raises validation error.""" - if backend != "vllm": - pytest.skip( - reason="logprobs are currently available only for the vLLM backend" - ) - - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "top_logprobs": top_logprobs_value, - "max_tokens": 10, - }, - ) - - # Should raise validation error for any value when logprobs is not True - assert response.status_code == 400 - assert ( - "`top_logprobs` can only be used when `logprobs` is True" - in response.json()["detail"] - ) - - def test_chat_completions_top_logprobs_validation( - self, client, model: str, messages: List[dict] - ): - """Test that top_logprobs > 20 is rejected by schema validation.""" - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "logprobs": True, - "top_logprobs": 25, # Exceeds maximum of 20 - "max_tokens": 5, - }, - ) + @pytest.mark.skip(reason="Not Implemented Yet") + def test_request_logit_bias(self): + pass - # Should raise schema validation error - assert response.status_code == 422 - assert "Input should be less than or equal to 20" in str( - response.json()["detail"] - ) + @pytest.mark.skip(reason="Not Implemented Yet") + def test_usage_response(self): + pass # For tests that won't use the same pytest fixture for server startup across @@ -712,7 +528,7 @@ def test_chat_completions_no_tokenizer( json={"model": model, "messages": messages}, ) - assert response.status_code == 500 + assert response.status_code == 400 assert response.json()["detail"] == "Unknown tokenizer" def test_chat_completions_custom_tokenizer( @@ -726,7 +542,7 @@ def test_chat_completions_custom_tokenizer( # Tokenizers can be provided by a local file path to a directory containing # the relevant files such as tokenizer.json and tokenizer_config.json. custom_tokenizer_path = str(Path(__file__).parent / "custom_tokenizer") - download_cmd = f"hf download --local-dir {custom_tokenizer_path} {tokenizer_model} --include *.json" + download_cmd = f"huggingface-cli download --local-dir {custom_tokenizer_path} {tokenizer_model} --include *.json" print(f"Running download command: {download_cmd}") subprocess.run(download_cmd.split(), check=True) @@ -741,12 +557,7 @@ def test_chat_completions_custom_tokenizer( responses = [] with TestClient(app_local) as client_local, TestClient(app_hf) as client_hf: - payload = { - "model": model, - "messages": messages, - "temperature": 0, - "seed": 0, - } + payload = {"model": model, "messages": messages, "temperature": 0} responses.append(client_local.post("/v1/chat/completions", json=payload)) responses.append(client_hf.post("/v1/chat/completions", json=payload)) @@ -799,7 +610,7 @@ def test_chat_completions_invalid_chat_tokenizer( json={"model": model, "messages": messages}, ) - assert response.status_code == 500 + assert response.status_code == 400 # Error may vary based on transformers version expected_errors = [ "cannot use apply_chat_template()", diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 9d496476e1..d89ff4701e 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,9 +45,8 @@ def test_completions_defaults(self, client, model: str, prompt: str): # NOTE: Could be improved to look for certain quality of response, # or tested with dummy identity model. assert response.json()["choices"][0]["text"].strip() - - usage = response.json().get("usage") - assert usage is not None + # "usage" currently not supported + assert not response.json()["usage"] @pytest.mark.parametrize( "sampling_parameter, value", @@ -57,6 +56,7 @@ def test_completions_defaults(self, client, model: str, prompt: str): ("top_p", 0.9), ("frequency_penalty", 0.5), ("presence_penalty", 0.2), + ("best_of", 1), ("n", 1), # logprobs is an integer for completions ("logprobs", 5), @@ -80,23 +80,10 @@ def test_completions_sampling_parameters( print("Response:", response.json()) # FIXME: Add support and remove this check - unsupported_parameters = ["logit_bias"] + unsupported_parameters = ["logprobs", "logit_bias"] if sampling_parameter in unsupported_parameters: assert response.status_code == 400 - assert response.json()["detail"] == "logit bias is not supported" - return - - # TRT-LLM backend doesn't support logprobs - if ( - sampling_parameter == "logprobs" - and value is not None - and model == "tensorrt_llm_bls" - ): - assert response.status_code == 400 - assert ( - "logprobs are currently available only for the vLLM backend" - in response.json()["detail"] - ) + assert response.json()["detail"] == "logit bias and log probs not supported" return assert response.status_code == 200 @@ -358,12 +345,7 @@ def test_no_prompt(self, client, model: str): ], ) def test_completions_multiple_choices( - self, - client, - sampling_parameter_dict: dict, - backend: str, - model: str, - prompt: str, + self, client, sampling_parameter_dict: dict, model: str, prompt: str ): response = client.post( "/v1/completions", @@ -374,11 +356,7 @@ def test_completions_multiple_choices( # FIXME: Add support and test for success # Expected to fail when n or best_of > 1, only single choice supported for now assert response.status_code == 400 - if backend == "vllm" and "best_of" in sampling_parameter_dict: - error_message = "best_of is no longer supported in vLLM backend" - else: - error_message = "only single choice" - assert error_message in response.json()["detail"] + assert "only single choice" in response.json()["detail"] @pytest.mark.skip(reason="Not Implemented Yet") def test_lora(self): @@ -388,128 +366,6 @@ def test_lora(self): def test_multi_lora(self): pass - @pytest.mark.parametrize("echo", [False, True]) - def test_echo(self, client, model: str, prompt: str, echo: bool): - response = client.post( - "/v1/completions", json={"model": model, "prompt": prompt, "echo": echo} - ) - - response_text = response.json()["choices"][0]["text"].strip() - if echo: - assert response_text.startswith(prompt) - else: - # TODO: Consider using a different prompt. In TRT-LLM model, the second response may contain the prompt in the middle of the response even if echo is False, e.g. " Briefly explained.\nWhat is machine learning? She learns from data\nmachine learning". - assert prompt not in response_text - - def test_usage_response(self, client, model: str, prompt: str): - response = client.post( - "/v1/completions", - json={"model": model, "prompt": prompt}, - ) - - assert response.status_code == 200 - usage = response.json().get("usage") - assert usage is not None - assert isinstance(usage["prompt_tokens"], int) - assert isinstance(usage["completion_tokens"], int) - assert isinstance(usage["total_tokens"], int) - assert usage["prompt_tokens"] > 0 - assert usage["completion_tokens"] > 0 - assert ( - usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] - ) - - def test_completions_logprobs(self, client, backend: str, model: str, prompt: str): - """Test logprobs parameter for completions.""" - response = client.post( - "/v1/completions", - json={ - "model": model, - "prompt": prompt, - "logprobs": 3, - "max_tokens": 10, - }, - ) - - # Non-vLLM backends should raise an error - if backend != "vllm": - assert response.status_code == 400 - assert ( - "logprobs are currently available only for the vLLM backend" - in response.json()["detail"] - ) - return - - assert response.status_code == 200 - response_json = response.json() - - # Check that logprobs are present in the response - choice = response_json["choices"][0] - assert "logprobs" in choice - logprobs = choice["logprobs"] - - assert logprobs is not None - assert "text_offset" in logprobs - assert "token_logprobs" in logprobs - assert "tokens" in logprobs - assert "top_logprobs" in logprobs - - assert isinstance(logprobs["text_offset"], list) - assert isinstance(logprobs["token_logprobs"], list) - assert isinstance(logprobs["tokens"], list) - assert isinstance(logprobs["top_logprobs"], list) - - # All lists should have the same length - num_tokens = len(logprobs["tokens"]) - assert len(logprobs["text_offset"]) == num_tokens - assert len(logprobs["token_logprobs"]) == num_tokens - assert len(logprobs["top_logprobs"]) == num_tokens - - # Validate each token - for i in range(num_tokens): - assert isinstance(logprobs["tokens"][i], str) - assert isinstance(logprobs["token_logprobs"][i], (int, float)) - assert isinstance(logprobs["text_offset"][i], int) - assert isinstance(logprobs["top_logprobs"][i], dict) - - # Validate top_logprobs dict contains token -> logprob mappings - for token, logprob in logprobs["top_logprobs"][i].items(): - assert isinstance(token, str) - assert isinstance(logprob, (int, float)) - - def test_completions_logprobs_zero(self, client, model: str, prompt: str): - """Test that logprobs=0 returns no logprobs.""" - response = client.post( - "/v1/completions", - json={ - "model": model, - "prompt": prompt, - "logprobs": 0, - "max_tokens": 10, - }, - ) - - assert response.status_code == 200 - response_json = response.json() - - # logprobs should be None when logprobs=0 - choice = response_json["choices"][0] - assert choice.get("logprobs") is None - - def test_completions_logprobs_validation(self, client, model: str, prompt: str): - """Test that logprobs > 5 is rejected by schema validation.""" - response = client.post( - "/v1/completions", - json={ - "model": model, - "prompt": prompt, - "logprobs": 7, # Exceeds maximum of 5 - "max_tokens": 5, - }, - ) - - # Should raise schema validation error - assert response.status_code == 422 - assert "Input should be less than or equal to 5" in str( - response.json()["detail"] - ) + @pytest.mark.skip(reason="Not Implemented Yet") + def test_usage_response(self): + pass diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py deleted file mode 100644 index 8462ff14a4..0000000000 --- a/python/openai/tests/test_embeddings.py +++ /dev/null @@ -1,600 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import base64 -import os -from pathlib import Path - -import numpy as np -import pytest - -# Results on A6000 GPU. The results vary slightly across GPU models. -EMBEDDING_OUTPUT_FLOAT = [ - -0.0365707763, - 0.076415509, - 0.0111756483, - 0.0361226462, - -0.0895985439, - 0.000943024294, - 0.0876693726, - -0.0594895333, - 0.0349566936, - -0.0937493294, - 0.132199496, - -0.000293674122, - 0.0232867915, - -0.022317145, - 0.00537066581, - -0.0994819254, - 0.0873948932, - -0.0594052449, - 0.0255441833, - -0.0896684974, - -0.05285028, - 0.0055408217, - -0.0262292251, - -0.0356511623, - 0.00121368305, - 0.0322951674, - 0.0308073629, - 0.0160015952, - -0.0187645368, - -0.00243343855, - -0.031794291, - -0.00363291171, - -0.0096142441, - 0.00763130654, - -0.0644499287, - -0.00359453214, - 0.0715369731, - -0.0835151896, - 0.0429252461, - -0.0022872088, - 0.0263325982, - -0.0283530336, - 0.0369597636, - 0.0226341616, - 0.0176856909, - -0.0551657975, - -0.0439660996, - -0.0230927691, - 0.0370879583, - -0.00998868514, - -0.0479186773, - 0.0299891923, - -0.0363422334, - -0.0903369784, - 0.0643197298, - 0.0223918613, - 0.0467988774, - -0.0922083259, - -0.0229205787, - 0.0192605518, - -0.0187863987, - -0.00113048754, - 0.0447385423, - 0.0520578064, - 0.0455449931, - -0.0587379821, - -0.0332143232, - -0.0847064033, - -0.0296805203, - 0.0522436313, - 0.0216157753, - -0.0455714688, - 0.0491720773, - -0.0230966564, - -0.0275325552, - -0.0460045785, - 0.000419082266, - -0.0763081461, - -0.106194891, - 0.0145287318, - -0.0292449873, - -0.0782747194, - -0.0399451442, - 0.0388121083, - -0.0387705714, - -0.0696005225, - 0.0101006646, - -0.0475178808, - 0.021968415, - 0.0317323506, - 0.0508495905, - -0.0438993014, - -0.0387620702, - -0.0523114018, - 0.0394089296, - -0.0469380654, - -0.0283870418, - -0.023771964, - 0.0526082292, - 0.079620786, - -0.00229133829, - 0.0339719504, - -0.0288904961, - -0.00747278007, - -0.00687310379, - 0.0226157606, - -0.0149615603, - 0.0292317756, - -0.0220453553, - -0.00164654257, - 0.00224191439, - 0.0798983052, - 0.0343615711, - 0.0089583965, - -0.0600032806, - -0.0543279201, - 0.00543989427, - -0.0362421572, - -0.0285114087, - 0.0446144193, - 0.0502739027, - 0.0574940592, - -0.0596095882, - 0.0327727199, - -0.117024891, - -0.0314645469, - 0.136668995, - 0.0, - -0.0752000064, - -0.0291419327, - 0.0460730754, - -0.0266116355, - 0.130121678, - 0.0257908553, - -0.035819497, - 0.0506630391, - -0.0240160581, - 0.0101758447, - 0.0496345982, - -0.0651563033, - -0.0362319537, - 0.00318966783, - 0.0281732827, - 0.0163737591, - -0.0164846163, - 0.0995664597, - 0.0733684897, - 0.00946230628, - -0.00913426094, - -0.0742818192, - 0.06195971, - 0.00591040403, - -0.0686667934, - 0.0317408852, - -0.0137725631, - 0.00940212607, - 0.0279442761, - -0.0201757029, - 0.0324154049, - 0.0228322521, - 0.00267707417, - 0.0171691254, - 0.00419936981, - -0.0130571416, - -0.0173847061, - -0.0758403093, - -0.0392607562, - -0.0166449342, - -0.00168224983, - 0.0199382622, - 0.0526281521, - 0.0502447523, - -0.129478946, - 0.0623795167, - -0.076566115, - 0.0532120988, - 0.0317721888, - -0.0135619631, - -0.0320665911, - -0.0216510575, - 0.106540799, - 0.0604757369, - -0.042855531, - 0.0153838852, - 0.0363844968, - 0.0429414809, - 0.0226413272, - -0.039562691, - 0.0454653203, - 0.0883121043, - 0.0196657199, - -0.0574896857, - 0.00670713792, - -0.018773403, - 0.000347356487, - 0.0141483406, - 0.0345573537, - 0.042631086, - -0.0191322975, - 0.0126261655, - 0.00105785835, - -0.00561600132, - -0.033773981, - 0.0439476408, - -0.0444635749, - -0.035605859, - 0.0268215071, - -0.0402722172, - 0.0911638364, - 0.00135396153, - 0.0485091805, - -0.0246936437, - -0.0408962481, - 0.0829341561, - -0.0306067225, - -0.125902891, - 0.0731693059, - 0.0934899077, - -0.102206372, - 0.0298629422, - 0.0766771212, - -0.0273114517, - -0.024197204, - 0.0, - -0.0244167317, - 0.0970985219, - -0.0935180783, - 0.0111901015, - -0.00198357552, - -0.0769073963, - -0.121803105, - -0.0520681925, - -0.0417099819, - 0.0248719398, - -0.0454342291, - -0.0240311194, - 0.0443824418, - -0.0372257456, - -0.0205930769, - 0.0910829455, - 0.0527612604, - 0.0190431513, - -0.014913708, - 0.0355940796, - -0.00282354676, - 0.0349615514, - 0.020905517, - 0.0863897428, - -0.0470590331, - 0.0979593918, - 0.0291745439, - 0.0513898097, - -0.165631235, - -0.0391068347, - 0.0751543418, - -0.043094065, - 0.028035799, - -0.0311034638, - 0.000913328957, - 0.0944983289, - -0.0198726766, - -0.0259133391, - -0.0108968485, - 0.0387883037, - 0.0520114712, - -0.0322841145, - 0.0131428279, - 0.0847168788, - 0.0164655242, - -0.0242556836, - -0.0100616785, - -0.066205658, - 0.0352719873, - -0.0125291245, - 0.00525686424, - -0.0127795609, - -0.025437668, - -0.0697134733, - -0.0109580038, - 0.00588004105, - 0.0470480993, - -0.0047857468, - 0.0171248112, - -0.0650963187, - -0.0638555363, - -6.33986347e-05, - 0.0477961302, - 0.0663475767, - 0.0779689029, - 0.0126418332, - 0.0279133115, - -0.0708932728, - -0.0341963358, - 0.0108084194, - -0.0322745182, - 0.0595393293, - 0.0120282508, - 0.0222520698, - -0.0312622078, - -0.00225326256, - -0.0878927261, - 0.0264401436, - 0.0213097129, - -0.0696384162, - -0.0348444693, - -0.0397011451, - -0.000856154773, - 0.0166215692, - -0.0223583411, - 0.0652333274, - 0.0340826549, - -0.0526116341, - -0.0165710896, - 0.0189326275, - -0.0277767386, - -0.0210060794, - -0.0209114067, - 0.004393938, - 0.022899719, - -1.13862484e-08, - 0.0633643791, - -0.00489465985, - -0.0535186455, - 0.066366978, - 0.00781527814, - -0.066619873, - 0.0434749424, - -0.008214131, - 0.00729982974, - 0.108025722, - -0.179152384, - 0.0326937772, - 0.0249010883, - 0.0834656358, - 0.0171424076, - 0.0380688161, - 0.0632147491, - -0.0202965494, - -0.0543595888, - 0.053911671, - 0.0329034328, - 0.0403351337, - -0.0204342771, - -0.0667905807, - 0.0286556948, - 0.00270259427, - -0.0699809119, - 0.0458261557, - -0.0122208307, - 0.0477884784, - 0.00767229684, - -0.0723900646, - -0.0811463594, - 0.0289416574, - 0.0698303133, - 0.0109635908, - -0.066716738, - -0.0869814679, - 0.0781401545, - -0.0747744292, - -0.0933830217, - 0.0906731561, - -0.118223637, - -0.00360242673, - 0.00453409506, - 0.0433466882, - -0.0145340748, - 0.101847678, - -0.0519261472, - 0.0441147573, - -0.0348532163, - 0.0241619237, - 0.0494029559, - -0.0146116838, - 0.0442838222, - -0.0998176262, - 0.0255751535, - -0.00209640572, - 0.0171953607, - 0.0489534624, - 0.0367930681, - 0.0853904262, - -0.0312620848, - -0.0702058449, -] - - -@pytest.mark.skipif( - os.environ.get("IMAGE_KIND") == "TRTLLM", - reason="TRT-LLM backend does not support embedding requests", -) -class TestEmbeddings: - @pytest.fixture(scope="class") - def client(self, fastapi_client_class_scope): - yield fastapi_client_class_scope - - @pytest.fixture(scope="class") - def model(self): - # Override with embeddings-specific model - return "all-MiniLM-L6-v2" - - @pytest.fixture(scope="class") - def tokenizer_model(self): - return None - - @pytest.fixture(scope="class") - def model_repository(self): - # Override with embeddings-specific repository - return str(Path(__file__).parent / "vllm_embedding_models") - - @pytest.fixture(scope="class") - def input(self): - return "The food was delicious and the waiter..." - - def _check_embedding_response(self, response, model, encoding_format="float"): - assert response.status_code == 200, response.json() - embedding = response.json()["data"][0]["embedding"] - assert embedding is not None - if encoding_format == "base64": - embedding = np.frombuffer(base64.b64decode(embedding), dtype=np.float32) - - # The results vary slightly across GPU models - result = np.allclose(EMBEDDING_OUTPUT_FLOAT, embedding, rtol=0, atol=1e-3) - assert ( - result - ), f"Embeddings do not match expected output\nExpect {EMBEDDING_OUTPUT_FLOAT},\ngot{embedding}" - - assert response.json()["data"][0]["object"] == "embedding" - assert response.json()["data"][0]["index"] == 0 - assert response.json()["model"] == model - - usage = response.json().get("usage") - assert usage is not None - assert usage["prompt_tokens"] == 12 - assert usage["total_tokens"] == 12 - - @pytest.mark.parametrize( - "input", - [ - "The food was delicious and the waiter...", - [101, 1996, 2833, 2001, 12090, 1998, 1996, 15610, 1012, 1012, 1012, 102], - ], - ) - def test_embeddings_defaults(self, client, model: str, input: str): - response = client.post( - "/v1/embeddings", - json={"model": model, "input": input}, - ) - - self._check_embedding_response(response, model) - - # FIXME: Python model cannot unload gracefully if raise error. - # def test_chat_completions_defaults( - # self, client, model: str, messages: List[dict], backend: str - # ): - # response = client.post( - # "/v1/chat/completions", - # json={"model": model, "messages": messages}, - # ) - - # assert response.status_code == 400 - # assert "does not support" in response.json()["detail"] - - @pytest.mark.parametrize( - "param_key, param_value", - [ - ("encoding_format", "invalid"), - ("encoding_format", 0), - ], - ) - def test_embeddings_invalid_parameters( - self, client, param_key, param_value, model: str, input: str - ): - response = client.post( - "/v1/embeddings", - json={ - "model": model, - "input": input, - param_key: param_value, - }, - ) - - # Assert schema validation error - assert response.status_code == 422, response.json() - - @pytest.mark.parametrize("encoding_format", ["float", "base64"]) - def test_embeddings_parameters( - self, client, encoding_format, model: str, input: str - ): - response = client.post( - "/v1/embeddings", - json={ - "model": model, - "input": input, - "encoding_format": encoding_format, - }, - ) - - self._check_embedding_response(response, model, encoding_format=encoding_format) - - def test_embeddings_empty_request(self, client): - response = client.post("/v1/embeddings", json={}) - assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "Field required" - - def test_embeddings_no_model(self, client, input: str): - response = client.post("/v1/embeddings", json={"input": input}) - assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "Field required" - - @pytest.mark.parametrize( - "model, error_code", - [ - ("", 400), - (123, 422), - ("Invalid", 400), - (None, 422), - ], - ) - def test_embeddings_invalid_model(self, client, model: str, input, error_code: int): - print("Model:", model) - # Message validation requires min_length of 1 - response = client.post("/v1/embeddings", json={"model": model, "input": input}) - assert response.status_code == error_code - if error_code == 400: - assert response.json()["detail"] == f"Unknown model: {model}" - else: - assert ( - response.json()["detail"][0]["msg"] == "Input should be a valid string" - ) - - def test_embeddings_no_input(self, client, model: str): - response = client.post("/v1/embeddings", json={"model": model}) - assert response.status_code == 422 - - @pytest.mark.parametrize( - "input", - [ - "", - [], - ], - ) - def test_embeddings_empty_input(self, client, model: str, input): - # Message validation requires min_length of 1 - response = client.post("/v1/embeddings", json={"model": model, "input": input}) - assert response.status_code == 422 - assert ( - response.json()["detail"][0]["msg"] - == "Value should have at least 1 item after validation, not 0" - ) - - @pytest.mark.parametrize( - "input", - [ - 123, - 1.5, - 0, - None, - ], - ) - def test_embeddings_invalid_input(self, client, model: str, input): - # Message validation requires min_length of 1 - response = client.post("/v1/embeddings", json={"model": model, "input": input}) - assert response.status_code == 422 - assert response.json()["detail"][0]["msg"] == "Input should be a valid string" diff --git a/python/openai/tests/test_lora.py b/python/openai/tests/test_lora.py deleted file mode 100644 index 0727cbd41f..0000000000 --- a/python/openai/tests/test_lora.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import os -import shutil -import unittest - -import pytest -from huggingface_hub import snapshot_download -from openai import BadRequestError, NotFoundError -from openai_frontend.engine.utils.triton import ( - _parse_lora_configs as parse_lora_configs, -) -from openai_frontend.engine.utils.triton import ( - _validate_lora_path_trtllm as validate_lora_path_trtllm, -) - -from .utils import OpenAIServer - - -def is_vllm_installed(): - try: - import vllm as _ - - return True - except ImportError: - return False - - -@pytest.mark.parametrize( - "model_repository,model_name,expect_error", - [ - ("openai_model_repository", "", True), # Empty string as model name. - ("openai_model_repository", " ", True), # Whitespace-only model name. - ("openai_model_repository", "invalid/path", True), - ("openai_model_repository", "invalid\\path", True), - ("openai_model_repository", "../outside/repo", True), - ("openai_model_repository", "../test_models/identity_py", True), - ("test_models", "../test_models/identity_py", True), - ("test_models", "identity_py", False), - ("test_models", "mock_llm", False), - ], -) -def test_parse_lora_configs(model_repository: str, model_name: str, expect_error: bool): - try: - parse_lora_configs(model_repository, model_name, 1, "vllm") - parse_lora_configs(model_repository, model_name, 1, "tensorrtllm") - except ValueError as e: - if expect_error: - assert ( - f"Invalid model name: '{model_name}'. Model names must be valid file-system-path segment names." - == str(e) - ) - else: - raise pytest.fail( - f"(model_repository='{model_repository}', model_name='{model_name}') raised ValueError unexpectedly: {e}" - ) - else: - if expect_error: - raise pytest.fail( - f"(model_repository='{model_repository}', model_name='{model_name}') did not raise ValueError as expected." - ) - - -@pytest.mark.skipif( - is_vllm_installed(), - reason="VLLM backend does not validate LoRA paths", -) -@pytest.mark.parametrize( - "lora_path,expect_error,error_message", - [ - # Valid relative path inside repo (requires .npy files to exist at runtime). - ("tensorrt_llm_bls/1/luotuo-lora-7b-0.1-weights", False, None), - ("tensorrt_llm_bls/1/Japanese-Alpaca-LoRA-7b-v0-weights", False, None), - # Absolute path not allowed. - ( - os.path.join( - os.path.abspath(os.curdir), - "tests/tensorrtllm_models", - "tensorrt_llm_bls/1/luotuo-lora-7b-0.1-weights", - ), - True, - f"must be a relative path inside its model repository", - ), - ("/etc/passwd", True, "must be a relative path inside its model repository"), - # Path outside repo (traversal). - ("tensorrt_llm_bls/1//../1/luotuo-lora-7b-0.1-weights", False, None), - ("../outside/lora", True, "must be inside its model repository"), - ("subdir/../../etc/passwd", True, "must be inside its model repository"), - # LoRA directory not found. - ("tensorrt_llm_bls/10", True, "LoRA directory 'tensorrt_llm_bls/10' not found"), - ( - "tensorrt_llm_bls/1/non_exist", - True, - "LoRA directory 'tensorrt_llm_bls/1/non_exist' not found", - ), - # LoRA file not found. - ("tensorrt_llm_bls/1", True, "LoRA file 'model.lora_weights.npy' not found"), - ], -) -def test_validate_lora_path_trtllm( - lora_path: str, - expect_error: bool, - error_message: str, -): - lora_name = "" - repo_path = "tests/tensorrtllm_models" - try: - validate_lora_path_trtllm(repo_path, lora_path, lora_name) - except Exception as e: - if not expect_error: - raise pytest.fail( - f"repo_path='{repo_path}' raised exception unexpectedly: {e}" - ) - assert error_message in str(e) - else: - if expect_error: - raise pytest.fail( - f"lora_path='{repo_path}' did not raise exception as expected." - ) - - -class LoRATest(unittest.TestCase): - _backend = "vllm" if is_vllm_installed() else "tensorrtllm" - _model_name = "gemma-2b" if _backend == "vllm" else "tensorrt_llm_bls" - # TODO: Find a LoRA model that has its own tokenizer. - _tokenizer = "meta-llama/Meta-Llama-3.1-8B-Instruct" - _lora_separator = "_lora_" - _prompt = "When was the wheel invented?" - # more prompts that may yield different outputs: - # - "Why can camels survive for long without water?" - # - "What is LAPR?" - # - "What is the difference between pets and cattle?" - _temperature = 0 - _top_p = 1 - - def _create_vllm_model_repository_with_lora(self): - shutil.rmtree("models", ignore_errors=True) - os.makedirs(f"models/{self._model_name}/1", exist_ok=True) - with open(f"models/{self._model_name}/config.pbtxt", "w") as f: - f.write('backend: "vllm"') - with open(f"models/{self._model_name}/1/model.json", "w") as f: - f.write( - json.dumps( - { - "model": "unsloth/gemma-2b", - "enable_lora": True, - "max_lora_rank": 32, - } - ) - ) - with open(f"models/{self._model_name}/1/multi_lora.json", "w") as f: - f.write( - json.dumps( - { - "doll": f"models/{self._model_name}/1/GemmaDoll", - "sheep": f"models/{self._model_name}/1/GemmaSheep", - } - ) - ) - snapshot_download( - repo_id="swathijn/GemmaDoll-2b-dolly-LORA-Tune", - local_dir=f"models/{self._model_name}/1/GemmaDoll", - ) - snapshot_download( - repo_id="eduardo-alvarez/GemmaSheep-2B-LORA-TUNED", - local_dir=f"models/{self._model_name}/1/GemmaSheep", - ) - - def _create_trtllm_model_repository_with_lora(self): - shutil.rmtree("models", ignore_errors=True) - shutil.copytree("tests/tensorrtllm_models", "models") - with open(f"models/{self._model_name}/1/multi_lora.json", "w") as f: - f.write( - json.dumps( - { - "doll": f"models/{self._model_name}/1/luotuo-lora-7b-0.1-weights", - "sheep": f"models/{self._model_name}/1/Japanese-Alpaca-LoRA-7b-v0-weights", - } - ) - ) - - def _create_vllm_model_repository_without_lora(self): - shutil.rmtree("models", ignore_errors=True) - os.makedirs(f"models/{self._model_name}/1", exist_ok=True) - with open(f"models/{self._model_name}/config.pbtxt", "w") as f: - f.write('backend: "vllm"') - with open(f"models/{self._model_name}/1/model.json", "w") as f: - f.write(json.dumps({"model": "unsloth/gemma-2b"})) - - def _create_trtllm_model_repository_without_lora(self): - shutil.rmtree("models", ignore_errors=True) - shutil.copytree("tests/tensorrtllm_models", "models") - - def _create_model_repository_mock_llm(self): - shutil.rmtree("models", ignore_errors=True) - os.makedirs(f"models/{self._model_name}/1", exist_ok=True) - with open(f"models/{self._model_name}/config.pbtxt", "w") as f: - f.write( - """ - backend: "python" - max_batch_size: 0 - model_transaction_policy { decoupled: True } - input [ - { - name: "text_input" - data_type: TYPE_STRING - dims: [ 1 ] - }, - { - name: "stream" - data_type: TYPE_BOOL - dims: [ 1 ] - }, - { - name: "sampling_parameters" - data_type: TYPE_STRING - dims: [ 1 ] - }, - { - name: "exclude_input_in_output" - data_type: TYPE_BOOL - dims: [ 1 ] - }, - { - name: "return_num_input_tokens" - data_type: TYPE_BOOL - dims: [1] - optional: true - }, - { - name: "return_num_output_tokens" - data_type: TYPE_BOOL - dims: [1] - optional: true - }, - { - name: "return_logprobs" - data_type: TYPE_BOOL - dims: [1] - optional: true - } - ] - output [ - { - name: "text_output" - data_type: TYPE_STRING - dims: [ -1 ] - } - ] - """ - ) - shutil.copy( - "tests/test_models/mock_llm/1/model.py", f"models/{self._model_name}/1" - ) - - def _get_model_name(self, lora_name): - model_name = self._model_name - if lora_name != "": - model_name += f"{self._lora_separator}{lora_name}" - return model_name - - def _test_list_models(self, client, expected_lora_names): - expected_model_names = [] - for lora_name in expected_lora_names: - expected_model_names.append(self._get_model_name(lora_name)) - models = client.models.list() - for model in models: - if self._backend == "tensorrtllm" and not model.id.startswith( - "tensorrt_llm_bls" - ): - continue - self.assertIn(model.id, expected_model_names) - expected_model_names.remove(model.id) - self.assertEqual( - len(expected_model_names), - 0, - f"expected_model_names: {expected_model_names}", - ) - - def _test_retrieve_model(self, client, lora_name): - model_name = self._get_model_name(lora_name) - model = client.models.retrieve(model_name) - self.assertEqual(model.id, model_name) - - def _test_completions(self, client, lora_name): - model_name = self._get_model_name(lora_name) - completion = client.completions.create( - model=model_name, - prompt=self._prompt, - temperature=self._temperature, - top_p=self._top_p, - ) - self.assertEqual(completion.model, model_name) - self.assertIsNotNone(completion.choices[0].text) - - def _test_chat_completion(self, client, lora_name): - model_name = self._get_model_name(lora_name) - messages = [{"role": "user", "content": self._prompt}] - chat_completion = client.chat.completions.create( - model=model_name, - messages=messages, - temperature=self._temperature, - top_p=self._top_p, - ) - self.assertEqual(chat_completion.model, model_name) - self.assertIsNotNone(chat_completion.choices[0].message.content) - - def test_lora_separator_not_set(self): - if self._backend == "vllm": - self._create_vllm_model_repository_with_lora() - elif self._backend == "tensorrtllm": - self._create_trtllm_model_repository_with_lora() - else: - raise Exception(f"Unexpected backend {self._backend=}") - - with OpenAIServer( - cli_args=[ - "--model-repository", - "models", - "--tokenizer", - self._tokenizer, - ], - env_dict={"CUDA_VISIBLE_DEVICES": "0"}, - ) as server: - client = server.get_client() - # Test listing/retrieving models - self._test_list_models(client, [""]) - self._test_retrieve_model(client, "") - with self.assertRaises(NotFoundError) as e: - self._test_retrieve_model(client, "doll") - expected_error = f"Error code: 404 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}doll'}}" - self.assertEqual(str(e.exception), expected_error) - with self.assertRaises(NotFoundError) as e: - self._test_retrieve_model(client, "sheep") - expected_error = f"Error code: 404 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}sheep'}}" - self.assertEqual(str(e.exception), expected_error) - # Test selecting LoRAs - self._test_completions(client, "") - self._test_chat_completion(client, "") - with self.assertRaises(BadRequestError) as e: - self._test_completions(client, "doll") - expected_error = f"Error code: 400 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}doll'}}" - self.assertEqual(str(e.exception), expected_error) - with self.assertRaises(BadRequestError) as e: - self._test_chat_completion(client, "sheep") - expected_error = f"Error code: 400 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}sheep'}}" - self.assertEqual(str(e.exception), expected_error) - - def test_lora_separator_set(self): - if self._backend == "vllm": - self._create_vllm_model_repository_with_lora() - elif self._backend == "tensorrtllm": - self._create_trtllm_model_repository_with_lora() - else: - raise Exception(f"Unexpected backend {self._backend=}") - - with OpenAIServer( - cli_args=[ - "--model-repository", - "models", - "--tokenizer", - self._tokenizer, - "--lora-separator", - self._lora_separator, - ], - env_dict={"CUDA_VISIBLE_DEVICES": "0"}, - ) as server: - client = server.get_client() - # Test listing/retrieving models - self._test_list_models(client, ["", "doll", "sheep"]) - self._test_retrieve_model(client, "") - self._test_retrieve_model(client, "doll") - self._test_retrieve_model(client, "sheep") - - # Test retrieving LoRAs unknown to the backend - with self.assertRaises(NotFoundError) as e: - self._test_retrieve_model(client, "unknown") - expected_error = f"Error code: 404 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}unknown'}}" - self.assertEqual(str(e.exception), expected_error) - - # Test selecting LoRAs - self._test_completions(client, "") - self._test_completions(client, "doll") - self._test_completions(client, "sheep") - self._test_chat_completion(client, "") - self._test_chat_completion(client, "doll") - self._test_chat_completion(client, "sheep") - - # Test selecting LoRAs unknown to the backend - expected_error = f"Error code: 400 - {{'detail': 'Unknown LoRA: unknown; for model: {self._model_name}{self._lora_separator}unknown'}}" - with self.assertRaises(BadRequestError) as e: - self._test_completions(client, "unknown") - self.assertEqual(str(e.exception), expected_error) - with self.assertRaises(BadRequestError) as e: - self._test_chat_completion(client, "unknown") - self.assertEqual(str(e.exception), expected_error) - - def test_lora_separator_set_for_lora_off_model(self): - if self._backend == "vllm": - self._create_vllm_model_repository_without_lora() - elif self._backend == "tensorrtllm": - self._create_trtllm_model_repository_without_lora() - else: - raise Exception(f"Unexpected backend {self._backend=}") - - with OpenAIServer( - cli_args=[ - "--model-repository", - "models", - "--tokenizer", - self._tokenizer, - "--lora-separator", - self._lora_separator, - ], - env_dict={"CUDA_VISIBLE_DEVICES": "0"}, - ) as server: - client = server.get_client() - # Test listing/retrieving models - self._test_list_models(client, [""]) - self._test_retrieve_model(client, "") - # Test retrieving models with LoRAs - with self.assertRaises(NotFoundError) as e: - self._test_retrieve_model(client, "doll") - expected_error = f"Error code: 404 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}doll'}}" - self.assertEqual(str(e.exception), expected_error) - # Test inference - self._test_completions(client, "") - self._test_chat_completion(client, "") - # Test selecting LoRAs - expected_error = f"Error code: 400 - {{'detail': 'Unknown LoRA: sheep; for model: {self._model_name}{self._lora_separator}sheep'}}" - with self.assertRaises(BadRequestError) as e: - self._test_completions(client, "sheep") - self.assertEqual(str(e.exception), expected_error) - with self.assertRaises(BadRequestError) as e: - self._test_chat_completion(client, "sheep") - self.assertEqual(str(e.exception), expected_error) - - @unittest.skipUnless(is_vllm_installed(), "vLLM not installed") - def test_lora_separator_set_for_non_vllm_formatted_models(self): - self._create_model_repository_mock_llm() - with OpenAIServer( - cli_args=[ - "--model-repository", - "models", - "--tokenizer", - self._tokenizer, - "--backend", - "vllm", - "--lora-separator", - self._lora_separator, - ], - env_dict={"CUDA_VISIBLE_DEVICES": "0"}, - ) as server: - client = server.get_client() - # Test listing/retrieving models - self._test_list_models(client, [""]) - self._test_retrieve_model(client, "") - # Test retrieving models with LoRAs - with self.assertRaises(NotFoundError) as e: - self._test_retrieve_model(client, "sheep") - expected_error = f"Error code: 404 - {{'detail': 'Unknown model: {self._model_name}{self._lora_separator}sheep'}}" - self.assertEqual(str(e.exception), expected_error) - # Test selecting LoRAs - # Expectation: - # If the frontend cannot determine which LoRA(s) are available, then any - # request with a well-formed LoRA model name will be inferenced. - self._test_completions(client, "doll") - self._test_chat_completion(client, "doll") - - -if __name__ == "__main__": - unittest.main() diff --git a/python/openai/tests/test_model_management.py b/python/openai/tests/test_model_management.py deleted file mode 100644 index 7cf420eed1..0000000000 --- a/python/openai/tests/test_model_management.py +++ /dev/null @@ -1,613 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import concurrent.futures -import http.client -import os -import random -import time -from pathlib import Path -from urllib.parse import urlparse - -import pytest -import requests -from tests.utils import OpenAIServer - -TEST_MODEL_REPOSITORY = str(Path(__file__).parent / "test_models") -TEST_MODEL = "mock_llm" -TEST_MODEL_2 = "identity_py" - - -# Test "--load-model" and "--model-control-mode" CLI options -@pytest.mark.openai -class TestModelControlCLIOptions: - @staticmethod - def _assert_server_launch_fails(args, expected_error: str): - """Helper: verify server fails to start and stderr contains expected_error.""" - with pytest.raises(Exception) as exc_info: - with OpenAIServer(args): - pass - assert expected_error in str(exc_info.value) - - def test_non_explicit_mode_load_model_error(self): - """--load-model without --model-control-mode=explicit must exit with error. - Error message matches native tritonserver exactly.""" - self._assert_server_launch_fails( - args=[ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--load-model", - TEST_MODEL, - ], - expected_error="Error: Use of '--load-model' requires setting '--model-control-mode=explicit' as well.", - ) - - def test_explicit_mode_load_zero_model(self): - args = [ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - ] - with OpenAIServer(args) as openai_server: - r = requests.get(openai_server.url_for("v1", "models"), timeout=10) - assert r.status_code == 200 - assert len(r.json()["data"]) == 0 - - def test_explicit_mode_load_one_model(self): - args = [ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - TEST_MODEL, - ] - with OpenAIServer(args) as openai_server: - r = requests.get(openai_server.url_for("v1", "models"), timeout=10) - names = [m["id"] for m in r.json()["data"]] - assert TEST_MODEL in names - assert TEST_MODEL_2 not in names - - def test_explicit_mode_load_multiple_models(self): - args = [ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - TEST_MODEL, - "--load-model", - TEST_MODEL_2, - ] - with OpenAIServer(args) as openai_server: - r = requests.get(openai_server.url_for("v1", "models"), timeout=10) - names = [m["id"] for m in r.json()["data"]] - assert TEST_MODEL in names - assert TEST_MODEL_2 in names - - def test_explicit_mode_load_all_models(self): - args = [ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - "*", - ] - with OpenAIServer(args) as openai_server: - r = requests.get(openai_server.url_for("v1", "models"), timeout=10) - names = [m["id"] for m in r.json()["data"]] - assert TEST_MODEL in names - assert TEST_MODEL_2 in names - - def test_explicit_mode_load_all_models_and_specific_model_error(self): - self._assert_server_launch_fails( - args=[ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - "*", - "--load-model", - TEST_MODEL, - ], - expected_error="Wildcard model name '*' must be the ONLY startup model if specified at all.", - ) - - def test_explicit_mode_load_nonexistent_model_error(self): - self._assert_server_launch_fails( - args=[ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - "nonexistent_model", - ], - expected_error="failed to poll model 'nonexistent_model': model not found in any model repository", - ) - - def test_explicit_mode_load_invalid_model_name_error(self): - invalid_model_names = [ - ( - os.path.relpath("/etc", TEST_MODEL_REPOSITORY), - "at least one version must be available under the version policy", - ), - ( - os.path.relpath("/etc/passwd", TEST_MODEL_REPOSITORY), - "Poll failed for model directory", - ), - ("model/..", "model not found in any model repository"), - ("..", "at least one version must be available under the version policy"), - ("/etc/passwd", "model not found in any model repository"), - ("", "Invalid model name"), - (" ", "model not found in any model repository"), - ("\n\t", "model not found in any model repository"), - ] - for model_name, expected_error in invalid_model_names: - self._assert_server_launch_fails( - args=[ - "--model-repository", - TEST_MODEL_REPOSITORY, - "--model-control-mode", - "explicit", - "--load-model", - model_name, - ], - expected_error=expected_error, - ) - - -class _ModelManagementBase: - @pytest.fixture(scope="class") - def base_url(self, server: OpenAIServer): - return server.url_root - - @pytest.fixture - def all_models(self) -> list[str]: - return [TEST_MODEL, TEST_MODEL_2] - - @pytest.fixture(scope="class") - def load_model(self) -> list[str]: - return [] - - @pytest.fixture(scope="class") - def server( - self, - model_repository: str, - tokenizer_model: str, - backend: str, - model_control_mode: str, - load_model: list[str], - ): - args = [ - "--model-repository", - model_repository, - ] - if tokenizer_model: - args += ["--tokenizer", tokenizer_model] - if backend: - args += ["--backend", backend] - if model_control_mode: - args += ["--model-control-mode", model_control_mode] - if load_model: - for model in load_model: - args += ["--load-model", model] - - with OpenAIServer(args) as openai_server: - yield openai_server - - @pytest.fixture(autouse=True) - def _cleanup(self, base_url, all_models: list[str]): - """Ensure clean state before and after every test by unloading the models.""" - for name in all_models: - requests.post(f"{base_url}/v1/models/{name}/unload") - yield - for name in all_models: - requests.post(f"{base_url}/v1/models/{name}/unload") - - @staticmethod - def _list_available_models(base_url: str) -> list[str]: - response = requests.get(f"{base_url}/v1/models") - assert response.status_code == 200 - return [m["id"] for m in response.json()["data"]] - - @staticmethod - def _assert_unknown_model(response: requests.Response): - assert response.status_code == 400 - assert "unknown model" in response.json()["detail"].lower() - - -class TestModelControlModeNone(_ModelManagementBase): - """Test NONE mode rejects load/unload API calls.""" - - @pytest.fixture(scope="class") - def model_repository(self): - return TEST_MODEL_REPOSITORY - - @pytest.fixture(scope="class") - def model_control_mode(self, request): - return request.param - - @pytest.mark.parametrize("model_control_mode", [None, "none"], indirect=True) - def test_load_and_unload_rejected(self, base_url): - """Test NONE mode rejects load/unload API calls.""" - - for api in ["load", "unload"]: - response = requests.post(f"{base_url}/v1/models/{TEST_MODEL}/{api}") - assert response.status_code == 400 - assert ( - "model load/unload requires --model-control-mode=explicit" - in response.json()["detail"].lower() - ) - - -class TestModelManagement(_ModelManagementBase): - """Test load/unload operations in EXPLICIT mode.""" - - @pytest.fixture(scope="class") - def model_control_mode(self): - return "explicit" - - @pytest.fixture(scope="class") - def model_repository(self): - return TEST_MODEL_REPOSITORY - - @staticmethod - def _assert_model_metadata(model_data: dict): - assert model_data["id"] - assert model_data["object"] == "model" - assert model_data["created"] > 0 - assert model_data["owned_by"] == "Triton Inference Server" - - def test_load_model(self, base_url): - # Server should start with no models loaded - assert self._list_available_models(base_url) == [] - - response = requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load") - assert response.status_code == 200 - self._assert_model_metadata(response.json()) - assert response.json()["id"] == TEST_MODEL - assert TEST_MODEL in self._list_available_models(base_url) - - response = requests.get(f"{base_url}/v1/models/{TEST_MODEL}") - assert response.status_code == 200 - self._assert_model_metadata(response.json()) - - def test_unload_model(self, base_url): - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load").status_code == 200 - ) - - response = requests.post(f"{base_url}/v1/models/{TEST_MODEL}/unload") - assert response.status_code == 200 - body = response.json() - assert body["status"] == "success" - assert body["model"] == TEST_MODEL - - assert TEST_MODEL not in self._list_available_models(base_url) - assert requests.get(f"{base_url}/v1/models/{TEST_MODEL}").status_code == 404 - - def test_load_rejects_duplicate(self, base_url): - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load").status_code == 200 - ) - - response = requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load") - assert response.status_code == 400 - assert "already loaded" in response.json()["detail"].lower() - - def test_load_unload_unknown_model(self, base_url): - response = requests.post(f"{base_url}/v1/models/unknown_model/load") - assert response.status_code == 500 - assert "failed to poll from model repository" in response.json()["detail"] - - response = requests.post(f"{base_url}/v1/models/unknown_model/unload") - self._assert_unknown_model(response) - - def test_load_unload_invalid_model_name(self, base_url): - invalid_model_names = [ - (os.path.relpath("/etc", TEST_MODEL_REPOSITORY), 404), - (os.path.relpath("/etc/passwd", TEST_MODEL_REPOSITORY), 404), - ("model/..", 404), - ("..", 400), - ("/etc/passwd", 404), - ("model/subdir", 404), - ("model/", 404), - ("", 404), - ("%20%20", 400), - ("%0A%09", 400), - ] - parsed = urlparse(base_url) - for model_name, expected_status in invalid_model_names: - for endpoint in ["load", "unload"]: - conn = http.client.HTTPConnection(parsed.hostname, parsed.port) - conn.request("POST", f"/v1/models/{model_name}/{endpoint}") - response = conn.getresponse() - - assert response.status == expected_status, ( - f"Expected {expected_status} for model name {model_name!r}, " - f"got {response.status} {response.read().decode()}" - ) - - def test_load_unload_reload(self, base_url): - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load").status_code == 200 - ) - assert TEST_MODEL in self._list_available_models(base_url) - - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/unload").status_code - == 200 - ) - assert TEST_MODEL not in self._list_available_models(base_url) - - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load").status_code == 200 - ) - assert TEST_MODEL in self._list_available_models(base_url) - - def test_load_multiple_models(self, base_url): - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load").status_code == 200 - ) - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL_2}/load").status_code - == 200 - ) - - names = self._list_available_models(base_url) - assert TEST_MODEL in names - assert TEST_MODEL_2 in names - - # Unload the first model - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/unload").status_code - == 200 - ) - names = self._list_available_models(base_url) - assert TEST_MODEL not in names - assert TEST_MODEL_2 in names - - # Unload the second model - assert ( - requests.post(f"{base_url}/v1/models/{TEST_MODEL_2}/unload").status_code - == 200 - ) - names = self._list_available_models(base_url) - assert TEST_MODEL not in names - assert TEST_MODEL_2 not in names - - -class TestConcurrentModelManagement(_ModelManagementBase): - """Test sequential and concurrent load/unload.""" - - @pytest.fixture(scope="class") - def model_repository(self): - return TEST_MODEL_REPOSITORY - - @pytest.fixture(scope="class") - def model_control_mode(self): - return "explicit" - - def test_concurrent_load_model(self, base_url): - futures = [] - concurrency = 5 - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - for _ in range(concurrency): - futures.append( - pool.submit( - requests.post, f"{base_url}/v1/models/{TEST_MODEL}/load" - ) - ) - - codes = sorted([future.result().status_code for future in futures]) - assert codes == [200] + [400] * ( - concurrency - 1 - ), f"Expected one 200 and the rest 400 for concurrent loads, got {codes}" - assert TEST_MODEL in self._list_available_models(base_url) - - def test_concurrent_unload_model(self, base_url): - # Load the model - requests.post(f"{base_url}/v1/models/{TEST_MODEL}/load") - assert TEST_MODEL in self._list_available_models(base_url) - - futures = [] - concurrency = 5 - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - for _ in range(concurrency): - futures.append( - pool.submit( - requests.post, f"{base_url}/v1/models/{TEST_MODEL}/unload" - ) - ) - - codes = sorted([future.result().status_code for future in futures]) - assert codes == [200] + [400] * ( - concurrency - 1 - ), f"Expected one 200 and the rest 400 for concurrent unloads, got {codes}" - assert TEST_MODEL not in self._list_available_models(base_url) - - def test_concurrent_load_multiple_models(self, base_url): - concurrency = 10 - tasks = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - for _ in range(concurrency // 2): - for model_name in [TEST_MODEL, TEST_MODEL_2]: - tasks.append( - pool.submit( - requests.post, f"{base_url}/v1/models/{model_name}/load" - ) - ) - - codes = sorted([task.result().status_code for task in tasks]) - assert codes == [200, 200] + [400] * ( - concurrency - 2 - ), f"Expected two 200s and the rest 400 for concurrent loads, got {codes}" - available_models = self._list_available_models(base_url) - assert TEST_MODEL in available_models - assert TEST_MODEL_2 in available_models - - def test_concurrent_unload_multiple_models(self, base_url): - # Load both models - for model_name in [TEST_MODEL, TEST_MODEL_2]: - requests.post(f"{base_url}/v1/models/{model_name}/load") - assert model_name in self._list_available_models(base_url) - - concurrency = 10 - tasks = [] - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - for _ in range(concurrency // 2): - for model_name in [TEST_MODEL, TEST_MODEL_2]: - tasks.append( - pool.submit( - requests.post, f"{base_url}/v1/models/{model_name}/unload" - ) - ) - - codes = sorted([task.result().status_code for task in tasks]) - assert codes == [200, 200] + [400] * ( - concurrency - 2 - ), f"Expected two 200s and the rest 400 for concurrent unloads, got {codes}" - available_models = self._list_available_models(base_url) - for model_name in [TEST_MODEL, TEST_MODEL_2]: - assert model_name not in available_models - - def test_concurrent_load_unload_stress(self, base_url): - futures = [] - concurrency = 50 - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as pool: - for _ in range(concurrency): - action = random.choice(["load", "unload"]) - model_name = random.choice([TEST_MODEL, TEST_MODEL_2]) - futures.append( - pool.submit( - requests.post, f"{base_url}/v1/models/{model_name}/{action}" - ) - ) - done, _ = concurrent.futures.wait( - futures, return_when=concurrent.futures.ALL_COMPLETED - ) - assert ( - len(done) == concurrency - ), f"Expected {concurrency} requests to be completed, got {len(done)}" - for future in done: - response = future.result() - assert response.status_code in ( - 200, - 400, - ), f"Unexpected status code: {response.status_code}" - - # Wait for server to be ready - for _ in range(10): - response = requests.get(f"{base_url}/health/ready") - if response.status_code == 200: - break - time.sleep(1) - assert response.status_code == 200 - - -# Test Inference with real LLM backend (vLLM / TRT-LLM) after load/unload -@pytest.mark.openai -class TestModelManagementInference(_ModelManagementBase): - @pytest.fixture(scope="class") - def model_control_mode(self): - return "explicit" - - @pytest.fixture(scope="class") - def load_model(self, model: str) -> list[str]: - # For tensorrt_llm_bls, we need to load all the dependent models - if model == "tensorrt_llm_bls": - return ["postprocessing", "preprocessing", "tensorrt_llm"] - - @staticmethod - def _assert_load(base_url, model_name: str): - r = requests.post( - f"{base_url}/v1/models/{model_name}/load", - ) - assert r.status_code == 200, f"Load {model_name} failed: {r.text}" - - @staticmethod - def _assert_unload(base_url, model_name: str): - r = requests.post( - f"{base_url}/v1/models/{model_name}/unload", - ) - assert r.status_code == 200, f"Unload {model_name} failed: {r.text}" - - @staticmethod - def _assert_usage(usage): - assert usage is not None - assert usage["prompt_tokens"] > 0 - assert usage["completion_tokens"] > 0 - assert ( - usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] - ) - - @staticmethod - def _completions(base_url, model_name: str, **kwargs): - return requests.post( - f"{base_url}/v1/completions", - json={ - "model": model_name, - "prompt": "What is machine learning?", - "max_tokens": 10, - **kwargs, - }, - ) - - def test_load_completions(self, base_url, model: str): - self._assert_unknown_model(self._completions(base_url, model)) - - self._assert_load(base_url, model) - r = self._completions(base_url, model) - assert r.status_code == 200 - data = r.json() - assert data["choices"][0]["text"].strip() - assert data["choices"][0]["finish_reason"] == "stop" - self._assert_usage(data["usage"]) - - self._assert_unload(base_url, model) - - def test_unload_rejects_inference(self, base_url, model: str): - self._assert_load(base_url, model) - assert self._completions(base_url, model).status_code == 200 - - self._assert_unload(base_url, model) - self._assert_unknown_model(self._completions(base_url, model)) - - def test_reload_inference(self, base_url, model: str): - self._assert_load(base_url, model) - assert self._completions(base_url, model).status_code == 200 - - self._assert_unload(base_url, model) - self._assert_unknown_model(self._completions(base_url, model)) - - self._assert_load(base_url, model) - r = self._completions(base_url, model) - assert r.status_code == 200 - assert r.json()["choices"][0]["text"].strip() diff --git a/python/openai/tests/test_models/mock_llm/config.pbtxt b/python/openai/tests/test_models/mock_llm/config.pbtxt index cac10b5de0..5f665ff543 100644 --- a/python/openai/tests/test_models/mock_llm/config.pbtxt +++ b/python/openai/tests/test_models/mock_llm/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,12 +41,6 @@ input [ name: "stream" data_type: TYPE_BOOL dims: [ 1, 1 ] - }, - { - name: "return_logprobs" - data_type: TYPE_BOOL - dims: [ 1, 1 ] - optional: true } ] diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 62fc6bfa18..6f1b456ab4 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,12 +26,10 @@ from typing import List -import numpy as np import openai import pytest -@pytest.mark.openai class TestOpenAIClient: @pytest.fixture(scope="class") def client(self, server): @@ -61,15 +59,6 @@ def test_openai_client_completion( assert completion.choices[0].text assert completion.choices[0].finish_reason == "stop" - usage = completion.usage - assert usage is not None - assert isinstance(usage.prompt_tokens, int) - assert isinstance(usage.completion_tokens, int) - assert isinstance(usage.total_tokens, int) - assert usage.prompt_tokens > 0 - assert usage.completion_tokens > 0 - assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - def test_openai_client_chat_completion( self, client: openai.OpenAI, model: str, messages: List[dict] ): @@ -82,26 +71,22 @@ def test_openai_client_chat_completion( assert chat_completion.choices[0].message.content assert chat_completion.choices[0].finish_reason == "stop" - usage = chat_completion.usage - assert usage is not None - assert isinstance(usage.prompt_tokens, int) - assert isinstance(usage.completion_tokens, int) - assert isinstance(usage.total_tokens, int) - assert usage.prompt_tokens > 0 - assert usage.completion_tokens > 0 - assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - @pytest.mark.parametrize("echo", [False, True]) def test_openai_client_completion_echo( - self, client: openai.OpenAI, echo: bool, model: str, prompt: str + self, client: openai.OpenAI, echo: bool, backend: str, model: str, prompt: str ): + if backend == "tensorrtllm": + pytest.skip( + reason="TRT-LLM backend currently only supports setting this parameter at model load time", + ) + completion = client.completions.create(prompt=prompt, model=model, echo=echo) + print(f"Completion results: {completion}") response = completion.choices[0].text if echo: - assert response.startswith(prompt) + assert prompt in response else: - # TODO: Consider using a different prompt. In TRT-LLM model, the second response may contain the prompt in the middle of the response even if echo is False, e.g. " Briefly explained.\nWhat is machine learning? She learns from data\nmachine learning". assert prompt not in response @pytest.mark.skip(reason="Not Implemented Yet") @@ -109,7 +94,6 @@ def test_openai_client_function_calling(self): pass -@pytest.mark.openai class TestAsyncOpenAIClient: @pytest.fixture(scope="class") def client(self, server): @@ -142,15 +126,6 @@ async def test_openai_client_completion( assert completion.choices[0].text assert completion.choices[0].finish_reason == "stop" - usage = completion.usage - assert usage is not None - assert isinstance(usage.prompt_tokens, int) - assert isinstance(usage.completion_tokens, int) - assert isinstance(usage.total_tokens, int) - assert usage.prompt_tokens > 0 - assert usage.completion_tokens > 0 - assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - @pytest.mark.asyncio async def test_openai_client_chat_completion( self, client: openai.AsyncOpenAI, model: str, messages: List[dict] @@ -162,16 +137,6 @@ async def test_openai_client_chat_completion( assert chat_completion.choices[0].message.content assert chat_completion.choices[0].finish_reason == "stop" - - usage = chat_completion.usage - assert usage is not None - assert isinstance(usage.prompt_tokens, int) - assert isinstance(usage.completion_tokens, int) - assert isinstance(usage.total_tokens, int) - assert usage.prompt_tokens > 0 - assert usage.completion_tokens > 0 - assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens - print(f"Chat completion results: {chat_completion}") @pytest.mark.asyncio @@ -185,7 +150,6 @@ async def test_completion_streaming( max_tokens=10, temperature=0.0, stream=False, - seed=0, ) output = chat_completion.choices[0].text stop_reason = chat_completion.choices[0].finish_reason @@ -197,7 +161,6 @@ async def test_completion_streaming( max_tokens=10, temperature=0.0, stream=True, - seed=0, ) chunks = [] finish_reason_count = 0 @@ -235,13 +198,13 @@ async def test_chat_streaming( seed = 0 temperature = 0.0 # Generate enough tokens to easily identify stop words are working. - max_completion_tokens = 64 + max_tokens = 64 # Test single chat completion for comparison chat_completion = await client.chat.completions.create( model=model, messages=messages, - max_completion_tokens=max_completion_tokens, + max_tokens=max_tokens, temperature=temperature, seed=seed, stream=False, @@ -254,7 +217,7 @@ async def test_chat_streaming( stream = await client.chat.completions.create( model=model, messages=messages, - max_completion_tokens=max_completion_tokens, + max_tokens=max_tokens, temperature=temperature, seed=seed, stream=True, @@ -270,7 +233,6 @@ async def test_chat_streaming( chunks.append(delta.content) if chunk.choices[0].finish_reason is not None: finish_reason_count += 1 - assert chunk.usage is None # finish reason should only return in last block assert finish_reason_count == 1 @@ -282,441 +244,7 @@ async def test_chat_streaming( streamed_output = "".join(chunks) assert streamed_output == output + @pytest.mark.skip(reason="Not Implemented Yet") @pytest.mark.asyncio - async def test_chat_streaming_usage_option( - self, client: openai.AsyncOpenAI, model: str, messages: List[dict] - ): - seed = 0 - temperature = 0.0 - max_tokens = 16 - - # Get usage and content from a non-streaming call - stream_false = await client.chat.completions.create( - model=model, - messages=messages, - max_tokens=max_tokens, - temperature=temperature, - seed=seed, - stream=False, - ) - usage_stream_false = stream_false.usage - stream_false_output = stream_false.choices[0].message.content - assert usage_stream_false is not None - assert stream_false_output is not None - - # First, run with include_usage=False. - stream_options_false = await client.chat.completions.create( - model=model, - messages=messages, - max_tokens=max_tokens, - temperature=temperature, - seed=seed, - stream=True, - stream_options={"include_usage": False}, - ) - chunks_false = [chunk async for chunk in stream_options_false] - for chunk in chunks_false: - assert chunk.usage is None, "Usage should be null when include_usage=False" - stream_options_false_output = "".join( - c.choices[0].delta.content - for c in chunks_false - if c.choices and c.choices[0].delta.content - ) - - # Now, run with include_usage=True. - stream_options_true = await client.chat.completions.create( - model=model, - messages=messages, - max_tokens=max_tokens, - temperature=temperature, - seed=seed, - stream=True, - stream_options={"include_usage": True}, - ) - chunks_true = [chunk async for chunk in stream_options_true] - content_chunks = [c for c in chunks_true if c.usage is None] - - # Verify that we received exactly one extra chunk. - assert len(chunks_true) == len(chunks_false) + 1 - - # Verify content is consistent - stream_options_true_output = "".join( - c.choices[0].delta.content - for c in content_chunks - if c.choices and c.choices[0].delta.content - ) - assert stream_options_true_output == stream_false_output - assert stream_options_true_output == stream_options_false_output - - # Verify the final chunk has usage data and empty choices. - final_chunk = chunks_true[-1] - assert final_chunk.usage is not None - assert len(final_chunk.choices) == 0 - usage_stream_options_true = final_chunk.usage - assert ( - isinstance(usage_stream_options_true.prompt_tokens, int) - and usage_stream_options_true.prompt_tokens > 0 - ) - assert ( - isinstance(usage_stream_options_true.completion_tokens, int) - and usage_stream_options_true.completion_tokens > 0 - ) - assert ( - usage_stream_options_true.total_tokens - == usage_stream_options_true.prompt_tokens - + usage_stream_options_true.completion_tokens - ) - - # Verify other chunks have no usage data. - for chunk in chunks_true[:-1]: - assert chunk.usage is None - - # Assert usage is consistent between streaming and non-streaming calls - assert usage_stream_false.model_dump() == usage_stream_options_true.model_dump() - - @pytest.mark.asyncio - async def test_completion_streaming_usage_option( - self, client: openai.AsyncOpenAI, model: str, prompt: str - ): - seed = 0 - temperature = 0.0 - max_tokens = 16 - - # Get usage and content from a non-streaming call - stream_false = await client.completions.create( - model=model, - prompt=prompt, - max_tokens=max_tokens, - temperature=temperature, - stream=False, - seed=seed, - ) - usage_stream_false = stream_false.usage - stream_false_output = stream_false.choices[0].text - assert usage_stream_false is not None - assert stream_false_output is not None - - # First, run with include_usage=False. - stream_options_false = await client.completions.create( - model=model, - prompt=prompt, - max_tokens=max_tokens, - temperature=temperature, - seed=seed, - stream=True, - stream_options={"include_usage": False}, - ) - chunks_false = [chunk async for chunk in stream_options_false] - for chunk in chunks_false: - assert chunk.usage is None - stream_options_false_output = "".join( - c.choices[0].text for c in chunks_false if c.choices and c.choices[0].text - ) - - # Now, run with include_usage=True. - stream_options_true = await client.completions.create( - model=model, - prompt=prompt, - max_tokens=max_tokens, - temperature=temperature, - stream=True, - seed=seed, - stream_options={"include_usage": True}, - ) - chunks_true = [chunk async for chunk in stream_options_true] - content_chunks = [c for c in chunks_true if c.usage is None] - - # Verify that we received exactly one extra chunk. - assert len(chunks_true) == len(chunks_false) + 1 - - # Verify content is consistent - stream_options_true_output = "".join( - c.choices[0].text for c in content_chunks if c.choices and c.choices[0].text - ) - assert stream_options_true_output == stream_false_output - assert stream_options_true_output == stream_options_false_output - - # Verify the final chunk has usage data and empty choices. - final_chunk = chunks_true[-1] - assert final_chunk.usage is not None - assert len(final_chunk.choices) == 0 - usage_stream_options_true = final_chunk.usage - assert ( - isinstance(usage_stream_options_true.prompt_tokens, int) - and usage_stream_options_true.prompt_tokens > 0 - ) - assert ( - isinstance(usage_stream_options_true.completion_tokens, int) - and usage_stream_options_true.completion_tokens > 0 - ) - assert ( - usage_stream_options_true.total_tokens - == usage_stream_options_true.prompt_tokens - + usage_stream_options_true.completion_tokens - ) - - # Verify other chunks have no usage data. - for chunk in chunks_true[:-1]: - assert chunk.usage is None - - # Assert usage is consistent between streaming and non-streaming calls - assert usage_stream_false.model_dump() == usage_stream_options_true.model_dump() - - @pytest.mark.asyncio - async def test_stream_options_without_streaming( - self, client: openai.AsyncOpenAI, model: str, prompt: str - ): - with pytest.raises(openai.BadRequestError) as e: - await client.completions.create( - model=model, - prompt=prompt, - stream=False, - stream_options={"include_usage": True}, - ) - assert "`stream_options` can only be used when `stream` is True" in str(e.value) - - with pytest.raises(openai.BadRequestError) as e: - await client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - stream=False, - stream_options={"include_usage": True}, - ) - assert "`stream_options` can only be used when `stream` is True" in str(e.value) - - @pytest.mark.asyncio - async def test_chat_completion_logprobs( - self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] - ): - """Test logprobs for chat completions and compare streaming vs non-streaming.""" - # Non-vLLM backends should raise an error - if backend != "vllm": - with pytest.raises(openai.BadRequestError) as exc_info: - await client.chat.completions.create( - model=model, - messages=messages, - logprobs=True, - top_logprobs=2, - max_tokens=10, - ) - assert "logprobs are currently available only for the vLLM backend" in str( - exc_info.value - ) - return - - # Test non-streaming - seed = 0 - temperature = 0.0 - chat_completion = await client.chat.completions.create( - model=model, - messages=messages, - logprobs=True, - top_logprobs=2, - max_tokens=10, - temperature=temperature, - seed=seed, - stream=False, - ) - - assert chat_completion.choices[0].message.content - assert chat_completion.choices[0].logprobs is not None - - logprobs = chat_completion.choices[0].logprobs - assert logprobs.content is not None - assert len(logprobs.content) > 0 - - # Validate each token logprob - for token_logprob in logprobs.content: - assert token_logprob.token - assert isinstance(token_logprob.logprob, float) - assert isinstance(token_logprob.bytes, list) - assert token_logprob.top_logprobs is not None - assert len(token_logprob.top_logprobs) > 0 - - # Test streaming and compare with non-streaming - stream = await client.chat.completions.create( - model=model, - messages=messages, - logprobs=True, - top_logprobs=2, - max_tokens=10, - temperature=temperature, - seed=seed, - stream=True, - ) - - chunks = [] - stream_logprobs = [] - async for chunk in stream: - if chunk.choices[0].delta.content: - chunks.append(chunk.choices[0].delta.content) - if chunk.choices[0].logprobs and chunk.choices[0].logprobs.content: - stream_logprobs.extend(chunk.choices[0].logprobs.content) - - # Assert streaming output matches non-streaming - streamed_output = "".join(chunks) - assert streamed_output == chat_completion.choices[0].message.content - - # Assert both streaming and non-streaming produce logprobs - assert len(stream_logprobs) > 0, "Streaming should produce logprobs" - assert len(stream_logprobs) == len(logprobs.content), "Same number of tokens" - - # Compare tokens and logprob values (using np.allclose for float comparison) - stream_tokens_list = [t.token for t in stream_logprobs] - non_stream_tokens_list = [t.token for t in logprobs.content] - stream_logprobs_values = [t.logprob for t in stream_logprobs] - non_stream_logprobs_values = [t.logprob for t in logprobs.content] - - assert stream_tokens_list == non_stream_tokens_list, "Tokens should match" - assert np.allclose( - stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-1 - ), "Logprob values should be close" - - @pytest.mark.asyncio - async def test_completion_logprobs( - self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str - ): - """Test logprobs for completions.""" - # Non-vLLM backends should raise an error - if backend != "vllm": - with pytest.raises(openai.BadRequestError) as exc_info: - await client.completions.create( - model=model, - prompt=prompt, - logprobs=3, - max_tokens=10, - ) - assert "logprobs are currently available only for the vLLM backend" in str( - exc_info.value - ) - return - - # Test non-streaming - seed = 0 - temperature = 0.0 - completion = await client.completions.create( - model=model, - prompt=prompt, - logprobs=3, - max_tokens=10, - temperature=temperature, - seed=seed, - stream=False, - ) - - assert completion.choices[0].text - assert completion.choices[0].logprobs is not None - - logprobs = completion.choices[0].logprobs - assert logprobs.tokens is not None - assert logprobs.token_logprobs is not None - assert logprobs.text_offset is not None - assert logprobs.top_logprobs is not None - - num_tokens = len(logprobs.tokens) - assert len(logprobs.token_logprobs) == num_tokens - assert len(logprobs.text_offset) == num_tokens - assert len(logprobs.top_logprobs) == num_tokens - - # Test streaming and compare with non-streaming - stream = await client.completions.create( - model=model, - prompt=prompt, - logprobs=3, - max_tokens=10, - temperature=temperature, - seed=seed, - stream=True, - ) - - chunks = [] - stream_tokens = [] - stream_token_logprobs = [] - stream_text_offsets = [] - stream_top_logprobs = [] - - async for chunk in stream: - if chunk.choices[0].text: - chunks.append(chunk.choices[0].text) - if chunk.choices[0].logprobs: - lp = chunk.choices[0].logprobs - if lp.tokens: - stream_tokens.extend(lp.tokens) - if lp.token_logprobs: - stream_token_logprobs.extend(lp.token_logprobs) - if lp.text_offset: - stream_text_offsets.extend(lp.text_offset) - if lp.top_logprobs: - stream_top_logprobs.extend(lp.top_logprobs) - - # Assert streaming output matches non-streaming - streamed_output = "".join(chunks) - assert streamed_output == completion.choices[0].text - - # Compare values (using np.allclose for float comparison) - assert stream_tokens == logprobs.tokens, "Tokens should match" - assert stream_text_offsets == logprobs.text_offset, "Text offsets should match" - assert stream_top_logprobs == logprobs.top_logprobs, "Top logprobs should match" - assert np.allclose( - stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-1 - ), "Token logprob values should be close" - - @pytest.mark.parametrize("top_logprobs_value", [0, 5]) - @pytest.mark.asyncio - async def test_top_logprobs_requires_logprobs( - self, - client: openai.AsyncOpenAI, - model: str, - messages: List[dict], - top_logprobs_value: int, - backend: str, - ): - """ - Test that top_logprobs without logprobs raises an error - """ - if backend != "vllm": - pytest.skip( - reason="logprobs are currently available only for the vLLM backend" - ) - - with pytest.raises(openai.BadRequestError) as exc_info: - await client.chat.completions.create( - model=model, - messages=messages, - top_logprobs=top_logprobs_value, # Without logprobs=True - max_tokens=5, - ) - assert "`top_logprobs` can only be used when `logprobs` is True" in str( - exc_info.value - ) - - @pytest.mark.asyncio - async def test_chat_top_logprobs_exceeds_max( - self, client: openai.AsyncOpenAI, model: str, messages: List[dict] - ): - """Test that top_logprobs > 20 raises schema validation error.""" - with pytest.raises(openai.UnprocessableEntityError) as exc_info: - await client.chat.completions.create( - model=model, - messages=messages, - logprobs=True, - top_logprobs=25, # Exceeds maximum of 20 - max_tokens=5, - ) - # Pydantic validation error - assert "less than or equal to 20" in str(exc_info.value).lower() - - @pytest.mark.asyncio - async def test_completion_logprobs_exceeds_max( - self, client: openai.AsyncOpenAI, model: str, prompt: str - ): - """Test that logprobs > 5 raises schema validation error.""" - with pytest.raises(openai.UnprocessableEntityError) as exc_info: - await client.completions.create( - model=model, - prompt=prompt, - logprobs=7, # Exceeds maximum of 5 - max_tokens=5, - ) - # Pydantic validation error - assert "less than or equal to 5" in str(exc_info.value).lower() + async def test_openai_client_function_calling(self): + pass diff --git a/python/openai/tests/test_openai_restricted_apis.py b/python/openai/tests/test_openai_restricted_apis.py deleted file mode 100755 index 200412df34..0000000000 --- a/python/openai/tests/test_openai_restricted_apis.py +++ /dev/null @@ -1,467 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from pathlib import Path -from typing import Dict, List, Optional - -import pytest -import requests -from tests.utils import OpenAIServer - - -def assert_response_success( - response: requests.Response, expected_status: int = 200, description: str = "" -): - """Assert that a response was successful.""" - assert ( - response.status_code == expected_status - ), f"{description} should return {expected_status}, got {response.status_code} {response.text}" - - -def assert_response_unauthorized( - response: requests.Response, expected_status: int = 401, description: str = "" -): - """Assert that a response was unauthorized.""" - assert ( - response.status_code == expected_status - ), f"{description} should be unauthorized with {expected_status}, got {response.status_code} {response.text}" - - -def make_get_request( - base_url: str, - endpoint: str, - headers: Optional[Dict[str, str]] = None, - timeout: int = 10, -): - """Make a GET request to the specified endpoint.""" - url = f"{base_url}{endpoint}" - response = requests.get(url, headers=headers, timeout=timeout) - return response - - -def verify_inference_endpoints( - base_url, model, headers, expected_success, description_prefix -): - def make_chat_request( - base_url: str, - model: str, - messages: List[Dict[str, str]], - headers: Optional[Dict[str, str]] = None, - max_tokens: int = 10, - timeout: int = 10, - ): - """Make a POST request to the chat completions endpoint.""" - url = f"{base_url}/v1/chat/completions" - data = { - "model": model, - "messages": messages, - "max_tokens": max_tokens, - } - response = requests.post(url, json=data, headers=headers, timeout=timeout) - return response - - def make_completion_request( - base_url: str, - model: str, - prompt: str, - headers: Optional[Dict[str, str]] = None, - max_tokens: int = 10, - timeout: int = 10, - ): - """Make a POST request to the completions endpoint.""" - url = f"{base_url}/v1/completions" - data = { - "model": model, - "prompt": prompt, - "max_tokens": max_tokens, - } - response = requests.post(url, json=data, headers=headers, timeout=timeout) - return response - - messages = [{"role": "user", "content": "Hello"}] - response = make_chat_request(base_url, model, messages, headers=headers) - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Chat completions endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Chat completions endpoint" - ) - - prompt = "Hello" - response = make_completion_request(base_url, model, prompt, headers=headers) - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Completions endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Completions endpoint" - ) - - -def verify_model_repository_endpoints( - base_url, model, headers, expected_success, description_prefix -): - # Verify model repository endpoints - response = make_get_request(base_url, "/v1/models", headers=headers) - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Models endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Models endpoint" - ) - - response = make_get_request(base_url, f"/v1/models/{model}", headers=headers) - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Specific model endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Specific model endpoint" - ) - - # Verify model management endpoints - for endpoint in ["unload", "load"]: - response = requests.post( - f"{base_url}/v1/models/{model}/{endpoint}", headers=headers - ) - if expected_success: - assert_response_success( - response, - description=f"{description_prefix} - Model {endpoint} endpoint", - ) - else: - assert_response_unauthorized( - response, - description=f"{description_prefix} - Model {endpoint} endpoint", - ) - - -def verify_metrics_endpoint(base_url, headers, expected_success, description_prefix): - # Test metrics endpoint - response = make_get_request(base_url, "/metrics", headers=headers) - assert_response_success(response, description="Unrestricted Metrics endpoint") - - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Metrics endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Metrics endpoint" - ) - - -def verify_health_endpoint(base_url, headers, expected_success, description_prefix): - # Test health endpoint - response = make_get_request(base_url, "/health/ready", headers=headers) - if expected_success: - assert_response_success( - response, description=f"{description_prefix} Health endpoint" - ) - else: - assert_response_unauthorized( - response, description=f"{description_prefix} Health endpoint" - ) - - -@pytest.mark.openai -class TestRestrictedAPIInvalidArguments: - """Test cases for malformed --openai-restricted-api arguments.""" - - def _test_server_startup_failure( - self, - malformed_api_arg, - expected_error_pattern=None, - ): - """Helper method to test that server fails to start with malformed arguments.""" - args = [ - "--model-repository", - str( - Path(__file__).parent / f"test_models" - ), # Hardcode to simple models to speed up tests - ] - if type(malformed_api_arg[0]) == list: - for api_arg in malformed_api_arg: - args.append("--openai-restricted-api") - args.extend(api_arg) - else: - args.append("--openai-restricted-api") - args.extend(malformed_api_arg) - - # Server should fail to start with malformed arguments - with pytest.raises((ValueError, Exception)) as exc_info: - with OpenAIServer(args) as openai_server: - pass # Should not reach here - - if expected_error_pattern: - assert expected_error_pattern in str( - exc_info.value - ), f"Expected error pattern '{expected_error_pattern}' not found in: {exc_info.value}" - - @pytest.mark.parametrize( - "malformed_arg", - [ - ["unknown-endpoint", "auth-key", "auth-value"], - ["invalid,inference", "auth-key", "auth-value"], # Mix of invalid and valid - ["inference,unknown", "auth-key", "auth-value"], # Mix of valid and invalid - ], - ) - def test_unknown_endpoint_names(self, malformed_arg): - """Test that server handles unknown endpoint names gracefully.""" - self._test_server_startup_failure( - malformed_arg, - expected_error_pattern="Unknown API", - ) - - @pytest.mark.parametrize( - "malformed_arg", - [ - ["inference,inference", "auth-key", "auth-value"], - ], - ) - def test_duplicate_apis(self, malformed_arg): - """Test that server handles duplicate APIs gracefully.""" - self._test_server_startup_failure( - malformed_arg, - expected_error_pattern="restricted api 'inference' can not be specified in multiple config groups", - ) - - @pytest.mark.parametrize( - "malformed_arg", - [ - # API with different auth specs - [ - ["inference", "auth-key1", "value1"], - ["inference", "auth-key2", "value2"], - ], - # API with same auth specs - [["inference", "auth-key", "value"], ["inference", "auth-key", "value"]], - # Multiple APIs with one duplicate - [ - ["inference", "auth-key1", "value1"], - ["model-repository", "auth-key2", "value2"], - ["inference", "auth-key3", "value3"], - ], - # All APIs duplicated - [ - ["inference", "auth-key1", "value1"], - ["model-repository", "auth-key2", "value2"], - ["inference", "auth-key3", "value3"], - ["model-repository", "auth-key4", "value4"], - ], - ], - ) - def test_conflict_configs(self, malformed_arg): - """Test that server fails when duplicate APIs are specified in multiple arguments.""" - # Test cases where the same API name appears in multiple --openai-restricted-api arguments - self._test_server_startup_failure( - malformed_arg, - expected_error_pattern="restricted api 'inference' can not be specified in multiple config groups", - ) - - -@pytest.mark.openai -class TestOpenAIServerRestrictedAPIs: - """Test cases for OpenAI server with restricted APIs functionality.""" - - @pytest.fixture(scope="class") - def server_with_restrictions(self, model_repository, tokenizer_model, backend): - """Start server with restricted APIs enabled.""" - args = [ - "--model-repository", - model_repository, - "--tokenizer", - tokenizer_model, - "--backend", - backend, - "--model-control-mode", - "explicit", - "--load-model", - "*", - "--openai-restricted-api", - "inference,model-repository", - "admin-key", - "admin-value", - ] - - with OpenAIServer(args) as openai_server: - yield openai_server - - @pytest.mark.parametrize( - "headers, expected_success, description", - [ - (None, False, "No auth"), - ({"admin-key": "admin-value"}, True, "Valid auth"), - ({"admin-key": "wrong-value"}, False, "Invalid auth value"), - ({"wrong-key": "admin-value"}, False, "Invalid auth key"), - ], - ) - def test_restricted_endpoints_with_auth( - self, server_with_restrictions, model, headers, expected_success, description - ): - """Test restricted endpoints with different authentication scenarios.""" - base_url = server_with_restrictions.url_root - - verify_model_repository_endpoints( - base_url, model, headers, expected_success, description - ) - verify_inference_endpoints( - base_url, model, headers, expected_success, description - ) - - def test_unrestricted_endpoints(self, server_with_restrictions): - """Test that unrestricted endpoints work without authentication.""" - base_url = server_with_restrictions.url_root - - verify_metrics_endpoint( - base_url, None, expected_success=True, description_prefix="Unrestricted" - ) - verify_health_endpoint( - base_url, None, expected_success=True, description_prefix="Unrestricted" - ) - - -@pytest.mark.openai -class TestOpenAIServerMultipleRestrictions: - """Test cases for OpenAI server with multiple restriction groups.""" - - @pytest.fixture(scope="class") - def server_multiple_restrictions(self, model_repository, tokenizer_model, backend): - """Start server with multiple restriction groups.""" - args = [ - "--model-repository", - model_repository, - "--tokenizer", - tokenizer_model, - "--backend", - backend, - "--model-control-mode", - "explicit", - "--load-model", - "*", - "--openai-restricted-api", - "model-repository", - "model-key", - "model-value", - "--openai-restricted-api", - "inference", - "infer-key", - "infer-value", - ] - - with OpenAIServer(args) as openai_server: - yield openai_server - - def test_endpoint_groups_with_correct_auth( - self, server_multiple_restrictions, model - ): - """Test that endpoint groups work with their specific authentication keys.""" - base_url = server_multiple_restrictions.url_root - - # Test model repository endpoints with model key - model_headers = {"model-key": "model-value"} - verify_model_repository_endpoints( - base_url, - model, - model_headers, - expected_success=True, - description_prefix="Correct model key", - ) - - # Test inference endpoints with inference key - infer_headers = {"infer-key": "infer-value"} - verify_inference_endpoints( - base_url, - model, - infer_headers, - expected_success=True, - description_prefix="Correct inference key", - ) - - @pytest.mark.parametrize( - "model_headers, model_description, infer_headers, infer_description", - [ - (None, "No auth", None, "No auth"), - ( - {"infer-key": "infer-value"}, - "Model key for inference endpoints", - {"model-key": "model-value"}, - "Inference key for model endpoints", - ), - ( - {"wrong-key": "wrong-value"}, - "Completely wrong key", - {"wrong-key": "wrong-value"}, - "Completely wrong key", - ), - ], - ) - def test_endpoint_groups_with_wrong_auth( - self, - server_multiple_restrictions, - model, - model_headers, - model_description, - infer_headers, - infer_description, - ): - """Test that endpoint groups are blocked with wrong authentication keys.""" - base_url = server_multiple_restrictions.url_root - - # Test scenarios where wrong auth keys are used - verify_model_repository_endpoints( - base_url, - model, - model_headers, - expected_success=False, - description_prefix=model_description, - ) - verify_inference_endpoints( - base_url, - model, - infer_headers, - expected_success=False, - description_prefix=infer_description, - ) - - def test_unrestricted_endpoints(self, server_multiple_restrictions): - """Test that unrestricted endpoints work without authentication.""" - base_url = server_multiple_restrictions.url_root - - verify_metrics_endpoint( - base_url, None, expected_success=True, description_prefix="Unrestricted" - ) - verify_health_endpoint( - base_url, None, expected_success=True, description_prefix="Unrestricted" - ) diff --git a/python/openai/tests/test_request_size.py b/python/openai/tests/test_request_size.py deleted file mode 100644 index 5670d00a7a..0000000000 --- a/python/openai/tests/test_request_size.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import asyncio -import json -import os -import sys -from pathlib import Path - -import pytest -import tritonserver -from fastapi.testclient import TestClient - -sys.path.append( - os.path.join(str(Path(__file__).resolve().parent.parent), "openai_frontend") -) - -from frontend.fastapi.middleware.request_size import RequestSizeLimitMiddleware -from tests.utils import setup_fastapi_app, setup_server -from utils.utils import HTTP_DEFAULT_MAX_INPUT_SIZE - -_MODEL = "mock_llm" - -# All POST endpoints. -_ENDPOINTS = ( - "/v1/chat/completions", - "/v1/completions", - "/v1/embeddings", - f"/v1/models/{_MODEL}/load", - f"/v1/models/{_MODEL}/unload", -) - - -@pytest.fixture(scope="module") -def client(): - """FastApiFrontend backed by a real Triton server with mock_llm loaded.""" - model_repository = str(Path(__file__).parent / "test_models") - server = setup_server( - model_repository, - model_control_mode=tritonserver.ModelControlMode.EXPLICIT, - load_models=[_MODEL], - ) - try: - app = setup_fastapi_app(tokenizer="", server=server, backend=None) - with TestClient(app) as test_client: - yield test_client - finally: - server.stop() - - -def _assert_content_too_large(response, actual_bytes: int) -> None: - assert response.status_code == 413 - body = response.json() - assert set(body) == {"error"} - error = body["error"] - assert error["type"] == "invalid_request_error" - assert error["code"] == "content_too_large" - assert error["message"] == RequestSizeLimitMiddleware._oversized_request_message( - actual_bytes, HTTP_DEFAULT_MAX_INPUT_SIZE - ) - - -class TestRequestSizeLimitMiddleware: - @pytest.mark.parametrize("endpoint", _ENDPOINTS) - def test_body_at_limit_is_not_rejected(self, client, endpoint): - response = client.post(endpoint, content=b"x" * HTTP_DEFAULT_MAX_INPUT_SIZE) - assert response.status_code != 413 - - @pytest.mark.parametrize("endpoint", _ENDPOINTS) - def test_body_over_limit_is_rejected(self, client, endpoint): - over = HTTP_DEFAULT_MAX_INPUT_SIZE + 1 - response = client.post(endpoint, content=b"x" * over) - _assert_content_too_large(response, over) - - @pytest.mark.parametrize("endpoint", _ENDPOINTS) - def test_chunked_body_over_limit_is_rejected(self, client, endpoint): - over = HTTP_DEFAULT_MAX_INPUT_SIZE + 1 - - # httpx switches to chunked transfer when content is an Iterable[bytes]. - def chunks(): - yield b"x" * HTTP_DEFAULT_MAX_INPUT_SIZE - yield b"x" - - response = client.post(endpoint, content=chunks()) - _assert_content_too_large(response, over) - - def test_get_without_body_is_unaffected(self, client): - response = client.get(_ENDPOINTS[0]) - assert response.status_code == 405 - - -class TestContentLengthValidation: - """Stage 1 rejects malformed Content-Length with 400.""" - - def _run_with_content_length(self, raw_value: bytes) -> tuple[int, dict]: - captured: dict = {"status": None, "body": b""} - - async def app(scope, receive, send): - raise AssertionError("app must not be reached for invalid Content-Length") - - async def receive(): - raise AssertionError("receive() must not be called when Stage 1 rejects") - - async def send(message): - if message["type"] == "http.response.start": - captured["status"] = message["status"] - elif message["type"] == "http.response.body": - captured["body"] += message.get("body", b"") - - middleware = RequestSizeLimitMiddleware( - app=app, http_max_input_size=HTTP_DEFAULT_MAX_INPUT_SIZE - ) - scope = { - "type": "http", - "method": "POST", - "path": "/v1/chat/completions", - "headers": [(b"content-length", raw_value)], - } - asyncio.run(middleware(scope, receive, send)) - - assert captured["status"] is not None, "no response was sent" - return captured["status"], json.loads(captured["body"]) - - def _assert_invalid_content_length(self, status: int, body: dict): - assert status == 400 - assert set(body) == {"error"} - error = body["error"] - assert error["type"] == "invalid_request_error" - assert error["code"] == "invalid_content_length" - - @pytest.mark.parametrize("raw", [b"not-a-number", b""]) - def test_non_integer_content_length_rejected_with_400(self, raw): - status, body = self._run_with_content_length(raw) - self._assert_invalid_content_length(status, body) - assert "not an integer" in body["error"]["message"] - - @pytest.mark.parametrize("raw", [b"-1", b"-1024"]) - def test_negative_content_length_rejected_with_400(self, raw): - status, body = self._run_with_content_length(raw) - self._assert_invalid_content_length(status, body) - assert "non-negative" in body["error"]["message"] diff --git a/python/openai/tests/test_tool_calling.py b/python/openai/tests/test_tool_calling.py deleted file mode 100644 index fccb27ac03..0000000000 --- a/python/openai/tests/test_tool_calling.py +++ /dev/null @@ -1,594 +0,0 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import json -import os -from typing import Dict, List, Optional - -import openai -import pytest -from openai.types.chat import ( - ChatCompletionMessageParam, - ChatCompletionMessageToolCall, - ChatCompletionNamedToolChoiceParam, - ChatCompletionToolParam, -) - -# resources for testing the tool callings -WEATHER_TOOL: ChatCompletionToolParam = { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to find the weather for, " - "e.g. 'San Francisco'", - }, - "state": { - "type": "string", - "description": "must the two-letter abbreviation for the state " - "that the city is in, e.g. 'CA' which would " - "mean 'California'", - }, - "unit": { - "type": "string", - "description": "The unit to fetch the temperature in", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["city", "state", "unit"], - }, - }, -} - -WEATHER_FORECAST_TOOL: ChatCompletionToolParam = { - "type": "function", - "function": { - "name": "get_n_day_weather_forecast", - "description": "Get an N-day weather forecast", - "parameters": { - "type": "object", - "properties": { - "city": { - "type": "string", - "description": "The city to find the weather for, " - "e.g. 'San Francisco'", - }, - "state": { - "type": "string", - "description": "must the two-letter abbreviation for the state " - "that the city is in, e.g. 'CA' which would " - "mean 'California'", - }, - "unit": { - "type": "string", - "description": "The unit to fetch the temperature in", - "enum": ["celsius", "fahrenheit"], - }, - "num_days": { - "type": "integer", - "description": "The number of days to forecast", - }, - }, - "required": ["city", "state", "unit", "num_days"], - }, - }, -} - -MESSAGES_ASKING_FOR_TOOLS: List[ChatCompletionMessageParam] = [ - { - "role": "system", - "content": "You're a helpful assistant! Answer the users question best you can.", - }, - {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"}, -] - -MESSAGES_WITH_TOOL_RESPONSE: List[ChatCompletionMessageParam] = [ - { - "role": "system", - "content": "You're a helpful assistant! Answer the users question best you can.", - }, - {"role": "user", "content": "What is the weather in Dallas, Texas in Fahrenheit?"}, - { - "role": "assistant", - "tool_calls": [ - { - "id": "123456789", - "type": "function", - "function": { - "name": "get_current_weather", - "arguments": '{"city": "Dallas", "state": "TX", ' - '"unit": "fahrenheit"}', - }, - } - ], - }, - {"role": "tool", "tool_call_id": "123456789", "content": "98"}, -] - -WEATHER_FORECAST_TOOL_CHOICE: ChatCompletionNamedToolChoiceParam = { - "function": {"name": "get_n_day_weather_forecast"}, - "type": "function", -} - - -@pytest.mark.openai -class TestAsyncClientToolCalling: - @pytest.fixture(scope="class") - def client(self, server): - return server.get_async_client() - - def validate_tool_calls_present( - self, tool_calls: Optional[List[ChatCompletionMessageToolCall]], skip_id=False - ): - assert tool_calls is not None - assert len(tool_calls) == 1 - assert tool_calls[0].type == "function" - assert tool_calls[0].function is not None - assert isinstance(tool_calls[0].id, str) - if not skip_id: - assert len(tool_calls[0].id) >= 9 - - def validate_weather_tool_arguments(self, parsed_arguments: Dict): - assert isinstance(parsed_arguments, Dict) - assert isinstance(parsed_arguments.get("city"), str) - assert isinstance(parsed_arguments.get("state"), str) - assert isinstance(parsed_arguments.get("unit"), str) - assert parsed_arguments.get("city") == "Dallas" - assert parsed_arguments.get("state") in ("TX", "Texas") - assert parsed_arguments.get("unit") == "fahrenheit" - - def validate_weather_forcast_tool_arguments(self, parsed_arguments: Dict): - assert isinstance(parsed_arguments, Dict) - assert isinstance(parsed_arguments.get("city"), str) - assert isinstance(parsed_arguments.get("state"), str) - assert isinstance(parsed_arguments.get("unit"), str) - assert isinstance(parsed_arguments.get("num_days"), int) - assert parsed_arguments.get("city") == "Dallas" - assert parsed_arguments.get("state") in ("TX", "Texas") - assert parsed_arguments.get("unit") == "fahrenheit" - - @pytest.mark.asyncio - async def test_tool_call_and_choice( - self, client: openai.AsyncOpenAI, model: str, backend: str - ): - # FIXME: [TRI-992] Maybe an issue on TRT-LLM but unverified. - if backend == "vllm" and model == "mistral-nemo-instruct-2407": - pytest.skip( - reason="Mistral model tool calling is not triggered with tool_choice=auto (default)." - ) - - chat_completion = await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - ) - - choice = chat_completion.choices[0] - stop_reason = chat_completion.choices[0].finish_reason - tool_calls = chat_completion.choices[0].message.tool_calls - - # make sure a tool call is present - self.validate_tool_calls_present(tool_calls) - assert stop_reason == "tool_calls" - - # make sure the weather tool was called (classic example) with arguments - assert tool_calls[0].function.name == WEATHER_TOOL["function"]["name"] - assert tool_calls[0].function.arguments is not None - assert isinstance(tool_calls[0].function.arguments, str) - - # make sure the arguments parse properly - parsed_arguments = json.loads(tool_calls[0].function.arguments) - self.validate_weather_tool_arguments(parsed_arguments) - - function_name: Optional[str] = None - function_args_str: str = "" - tool_call_id: Optional[str] = None - role_name: Optional[str] = None - finish_reason_count: int = 0 - - # make the same request, streaming - stream = await client.chat.completions.create( - model=model, - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - stream=True, - ) - - async for chunk in stream: - assert chunk.choices[0].index == 0 - - if chunk.choices[0].finish_reason: - finish_reason_count += 1 - assert chunk.choices[0].finish_reason == "tool_calls" - - # if a role is being streamed make sure it wasn't already set to - # something else - if chunk.choices[0].delta.role: - assert not role_name or role_name == "assistant" - role_name = "assistant" - - # if a tool call is streamed make sure there's exactly one - # (based on the request parameters - streamed_tool_calls = chunk.choices[0].delta.tool_calls - - if streamed_tool_calls and len(streamed_tool_calls) > 0: - assert len(streamed_tool_calls) == 1 - tool_call = streamed_tool_calls[0] - - # if a tool call ID is streamed, make sure one hasn't been already - if tool_call.id: - assert not tool_call_id - tool_call_id = tool_call.id - - # if parts of the function start being streamed - if tool_call.function: - # if the function name is defined, set it. it should be streamed - # IN ENTIRETY, exactly one time. - if tool_call.function.name: - assert function_name is None - assert isinstance(tool_call.function.name, str) - function_name = tool_call.function.name - if tool_call.function.arguments: - assert isinstance(tool_call.function.arguments, str) - function_args_str += tool_call.function.arguments - - assert finish_reason_count == 1 - assert role_name == "assistant" - assert isinstance(tool_call_id, str) and (len(tool_call_id) >= 9) - - # validate the name and arguments - assert function_name == WEATHER_TOOL["function"]["name"] - assert function_name == tool_calls[0].function.name - assert isinstance(function_args_str, str) - - # validate arguments - streamed_args = json.loads(function_args_str) - self.validate_weather_tool_arguments(streamed_args) - - # make sure everything matches non-streaming except for ID - assert function_name == tool_calls[0].function.name - assert choice.message.role == role_name - assert choice.message.tool_calls[0].function.name == function_name - - # compare streamed with non-streamed args Dict-wise, not string-wise - # because character-to-character comparison might not work e.g. the tool - # call parser adding extra spaces or something like that. we care about the - # dicts matching not byte-wise match - assert parsed_arguments == streamed_args - - @pytest.mark.asyncio - async def test_tool_call_with_reply_response( - self, client: openai.AsyncOpenAI, model: str, backend: str - ): - chat_completion = await client.chat.completions.create( - messages=MESSAGES_WITH_TOOL_RESPONSE, - temperature=0, - max_completion_tokens=128, - model=model, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - seed=0, - ) - - choice = chat_completion.choices[0] - - assert choice.finish_reason != "tool_calls" # "stop" - assert choice.message.role == "assistant" - assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0 - assert choice.message.content is not None - - stream = await client.chat.completions.create( - messages=MESSAGES_WITH_TOOL_RESPONSE, - temperature=0, - max_completion_tokens=128, - model=model, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - stream=True, - seed=0, - ) - - chunks: List[str] = [] - finish_reason_count = 0 - role_sent: bool = False - - async for chunk in stream: - delta = chunk.choices[0].delta - - if delta.role: - assert not role_sent - assert delta.role == "assistant" - role_sent = True - - if delta.content: - chunks.append(delta.content) - - if chunk.choices[0].finish_reason is not None: - finish_reason_count += 1 - assert chunk.choices[0].finish_reason == choice.finish_reason - - assert not delta.tool_calls or len(delta.tool_calls) == 0 - - assert role_sent - assert finish_reason_count == 1 - assert len(chunks) - - # validate if steaming and non-streaming generates the same content - assert "".join(chunks) == choice.message.content - - @pytest.mark.asyncio - async def test_tool_call_with_named_tool_choice( - self, client: openai.AsyncOpenAI, model: str - ): - chat_completion = await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tool_choice=WEATHER_FORECAST_TOOL_CHOICE, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - ) - - choice = chat_completion.choices[0] - stop_reason = chat_completion.choices[0].finish_reason - tool_calls = chat_completion.choices[0].message.tool_calls - - # make sure a tool call is present - self.validate_tool_calls_present(tool_calls, skip_id=True) - assert stop_reason != "tool_calls" - - # make sure the weather tool was called (classic example) with arguments - assert tool_calls[0].function.name == WEATHER_FORECAST_TOOL["function"]["name"] - assert tool_calls[0].function.arguments is not None - assert isinstance(tool_calls[0].function.arguments, str) - - # make sure the arguments parse properly - parsed_arguments = json.loads(tool_calls[0].function.arguments) - self.validate_weather_forcast_tool_arguments(parsed_arguments) - - function_name: Optional[str] = None - function_args_str: str = "" - tool_call_id: Optional[str] = None - role_name: Optional[str] = None - finish_reason_count: int = 0 - - # make the same request, streaming - stream = await client.chat.completions.create( - model=model, - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - tool_choice=WEATHER_FORECAST_TOOL_CHOICE, - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - stream=True, - ) - - async for chunk in stream: - assert chunk.choices[0].index == 0 - - if chunk.choices[0].finish_reason: - finish_reason_count += 1 - assert chunk.choices[0].finish_reason != "tool_calls" - - # if a role is being streamed make sure it wasn't already set to - # something else - if chunk.choices[0].delta.role: - assert not role_name or role_name == "assistant" - role_name = "assistant" - - # if a tool call is streamed make sure there's exactly one - # (based on the request parameters - streamed_tool_calls = chunk.choices[0].delta.tool_calls - - if streamed_tool_calls and len(streamed_tool_calls) > 0: - assert len(streamed_tool_calls) == 1 - tool_call = streamed_tool_calls[0] - - # if a tool call ID is streamed, make sure one hasn't been already - if tool_call.id: - assert not tool_call_id - tool_call_id = tool_call.id - - # if parts of the function start being streamed - if tool_call.function: - # if the function name is defined, set it. it should be streamed - # IN ENTIRETY, exactly one time. - if tool_call.function.name: - assert isinstance(tool_call.function.name, str) - function_name = tool_call.function.name - if tool_call.function.arguments: - assert isinstance(tool_call.function.arguments, str) - function_args_str += tool_call.function.arguments - - assert finish_reason_count == 1 - assert role_name == "assistant" - - # validate the name and arguments - assert function_name == WEATHER_FORECAST_TOOL["function"]["name"] - assert function_name == tool_calls[0].function.name - assert isinstance(function_args_str, str) - - # validate arguments - streamed_args = json.loads(function_args_str) - self.validate_weather_forcast_tool_arguments(streamed_args) - - # make sure everything matches non-streaming except for ID - assert function_name == tool_calls[0].function.name - assert choice.message.role == role_name - assert choice.message.tool_calls[0].function.name == function_name - - @pytest.mark.asyncio - async def test_tool_call_with_required_tool_choice( - self, client: openai.AsyncOpenAI, model: str - ): - chat_completion = await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tool_choice="required", - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - ) - - choice = chat_completion.choices[0] - stop_reason = chat_completion.choices[0].finish_reason - tool_calls = chat_completion.choices[0].message.tool_calls - - # make sure a tool call is present - self.validate_tool_calls_present(tool_calls, skip_id=True) - assert stop_reason != "tool_calls" - - # make sure the weather tool was called (classic example) with arguments - assert tool_calls[0].function.name == WEATHER_TOOL["function"]["name"] - assert tool_calls[0].function.arguments is not None - assert isinstance(tool_calls[0].function.arguments, str) - - # make sure the arguments parse properly - parsed_arguments = json.loads(tool_calls[0].function.arguments) - self.validate_weather_tool_arguments(parsed_arguments) - - function_name: Optional[str] = None - function_args_str: str = "" - tool_call_id: Optional[str] = None - role_name: Optional[str] = None - finish_reason_count: int = 0 - - # make the same request, streaming - stream = await client.chat.completions.create( - model=model, - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - tool_choice="required", - tools=[WEATHER_TOOL, WEATHER_FORECAST_TOOL], - logprobs=False, - stream=True, - ) - - async for chunk in stream: - assert chunk.choices[0].index == 0 - - if chunk.choices[0].finish_reason: - finish_reason_count += 1 - assert chunk.choices[0].finish_reason != "tool_calls" - - # if a role is being streamed make sure it wasn't already set to - # something else - if chunk.choices[0].delta.role: - assert not role_name or role_name == "assistant" - role_name = "assistant" - - # if a tool call is streamed make sure there's exactly one - # (based on the request parameters - streamed_tool_calls = chunk.choices[0].delta.tool_calls - - if streamed_tool_calls and len(streamed_tool_calls) > 0: - assert len(streamed_tool_calls) == 1 - tool_call = streamed_tool_calls[0] - - # if a tool call ID is streamed, make sure one hasn't been already - if tool_call.id: - assert not tool_call_id - tool_call_id = tool_call.id - - # if parts of the function start being streamed - if tool_call.function: - # if the function name is defined, set it. it should be streamed - # IN ENTIRETY, exactly one time. - if tool_call.function.name: - assert isinstance(tool_call.function.name, str) - function_name = tool_call.function.name - if tool_call.function.arguments: - assert isinstance(tool_call.function.arguments, str) - function_args_str += tool_call.function.arguments - - assert finish_reason_count == 1 - assert role_name == "assistant" - - # validate the name and arguments - assert function_name == WEATHER_TOOL["function"]["name"] - assert function_name == tool_calls[0].function.name - assert isinstance(function_args_str, str) - - # validate arguments - streamed_args = json.loads(function_args_str) - self.validate_weather_tool_arguments(streamed_args) - - # make sure everything matches non-streaming except for ID - assert function_name == tool_calls[0].function.name - assert choice.message.role == role_name - assert choice.message.tool_calls[0].function.name == function_name - - @pytest.mark.asyncio - async def test_inconsistent_tool_choice_and_tools( - self, client: openai.AsyncOpenAI, model: str - ): - # tool choice function but the tools are empty - with pytest.raises(openai.BadRequestError): - await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tool_choice=WEATHER_FORECAST_TOOL_CHOICE, - logprobs=False, - ) - # tool choice function that is not provided in the tools - with pytest.raises(openai.BadRequestError): - await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tool_choice=WEATHER_FORECAST_TOOL_CHOICE, - tools=[WEATHER_TOOL], - logprobs=False, - ) - - # tool choice required but tools is empty - with pytest.raises(openai.BadRequestError): - await client.chat.completions.create( - messages=MESSAGES_ASKING_FOR_TOOLS, - temperature=0, - max_completion_tokens=128, - model=model, - tool_choice="required", - tools=[], - logprobs=False, - ) diff --git a/python/openai/tests/utils.py b/python/openai/tests/utils.py index c73453ee90..fdffcc5ea9 100644 --- a/python/openai/tests/utils.py +++ b/python/openai/tests/utils.py @@ -1,4 +1,4 @@ -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,6 @@ import os import subprocess import sys -import threading import time from pathlib import Path from typing import Dict, List, Optional @@ -42,15 +41,9 @@ # TODO: Cleanup, refactor, mock, etc. -def setup_server( - model_repository: str, - model_control_mode: tritonserver.ModelControlMode = tritonserver.ModelControlMode.NONE, - load_models: Optional[List[str]] = None, -): +def setup_server(model_repository: str): server: tritonserver.Server = tritonserver.Server( model_repository=model_repository, - model_control_mode=model_control_mode, - startup_models=load_models or [], log_verbose=0, log_info=True, log_warn=True, @@ -59,17 +52,9 @@ def setup_server( return server -def setup_fastapi_app( - tokenizer: str, - server: tritonserver.Server, - backend: str, - default_max_tokens: int = 16, -): +def setup_fastapi_app(tokenizer: str, server: tritonserver.Server, backend: str): engine: TritonLLMEngine = TritonLLMEngine( - server=server, - tokenizer=tokenizer, - backend=backend, - default_max_tokens=default_max_tokens, + server=server, tokenizer=tokenizer, backend=backend ) frontend: FastApiFrontend = FastApiFrontend(engine=engine) return frontend.app @@ -78,7 +63,7 @@ def setup_fastapi_app( # Heavily inspired by vLLM's test infrastructure class OpenAIServer: API_KEY = "EMPTY" # Triton's OpenAI server does not need API key - START_TIMEOUT = 240 # wait for server to start for up to 240 seconds, mistral model takes longer time to start + START_TIMEOUT = 120 # wait for server to start for up to 120 seconds def __init__( self, @@ -100,29 +85,13 @@ def __init__( ["python3", script_path] + cli_args, env=env, stdout=sys.stdout, - stderr=subprocess.PIPE, # Capture stderr - text=True, + stderr=sys.stderr, ) - self.stderr_lines = [] - threading.Thread(target=self._read_stderr, daemon=True).start() # Wait until health endpoint is responsive self._wait_for_server( url=self.url_for("health", "ready"), timeout=self.START_TIMEOUT ) - def _read_stderr(self): - """Read stderr and print to console in real-time. Continues throughout server lifecycle.""" - try: - if self.proc.stderr: - for line in iter(self.proc.stderr.readline, ""): - self.stderr_lines.append(line.rstrip("\n\r")) - sys.stderr.write(line) - sys.stderr.flush() - except (OSError, ValueError, BrokenPipeError) as exc: - # Ignore expected errors during process shutdown, but log for debugging. - sys.stderr.write(f"[OpenAIServer] Error while reading stderr: {exc}\n") - sys.stderr.flush() - def __enter__(self): return self @@ -144,31 +113,11 @@ def _wait_for_server(self, *, url: str, timeout: float): except Exception as err: result = self.proc.poll() if result is not None and result != 0: - stderr_text = ( - "\n".join(self.stderr_lines) - if self.stderr_lines - else "No stderr output" - ) - error = RuntimeError( - f"Server exited unexpectedly with return code {result}.\n" - f"Stderr output:\n{stderr_text}" - ) - error.stderr_lines = list(self.stderr_lines) - raise error from err + raise RuntimeError("Server exited unexpectedly.") from err time.sleep(0.5) if time.time() - start > timeout: - stderr_text = ( - "\n".join(self.stderr_lines) - if self.stderr_lines - else "No stderr output" - ) - error = RuntimeError( - f"Server failed to start in time.\n" - f"Stderr output:\n{stderr_text}" - ) - error.stderr_lines = list(self.stderr_lines) - raise error from err + raise RuntimeError("Server failed to start in time.") from err @property def url_root(self) -> str: diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json deleted file mode 100644 index 2ad058c2e5..0000000000 --- a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json +++ /dev/null @@ -1 +0,0 @@ -{"model": "sentence-transformers/all-MiniLM-L6-v2", "gpu_memory_utilization": 0.5} diff --git a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json deleted file mode 100644 index b7ce0ee199..0000000000 --- a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json +++ /dev/null @@ -1 +0,0 @@ -{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.9} \ No newline at end of file diff --git a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/config.pbtxt b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/config.pbtxt deleted file mode 100644 index 39b3c48edb..0000000000 --- a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/config.pbtxt +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -backend: "vllm" -instance_group [{kind: KIND_MODEL}] diff --git a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json index df85a05da0..cb9b14c765 100644 --- a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json +++ b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json @@ -1 +1 @@ -{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.9} +{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "disable_log_requests": true, "gpu_memory_utilization": 0.9} diff --git a/qa/L0_backend_bls/test.sh b/qa/L0_backend_bls/test.sh index 9e27f50870..6210e7654c 100755 --- a/qa/L0_backend_bls/test.sh +++ b/qa/L0_backend_bls/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,7 +45,7 @@ apt update -q=2 \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ - && apt-get install -y --no-install-recommends cmake=4.0.3* cmake-data=4.0.3* \ + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* \ rapidjson-dev cmake --version @@ -57,7 +57,6 @@ git clone --single-branch --depth=1 -b $TRITON_BACKEND_REPO_TAG \ (cd backend/examples/backends/bls && mkdir build && cd build && - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ diff --git a/qa/L0_backend_identity/identity_test.py b/qa/L0_backend_identity/identity_test.py index b96649241d..a607e4189b 100755 --- a/qa/L0_backend_identity/identity_test.py +++ b/qa/L0_backend_identity/identity_test.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,7 +30,6 @@ import sys from builtins import range -import ml_dtypes import numpy as np import requests as httpreq import tritonclient.grpc as grpcclient @@ -202,8 +201,8 @@ ("identity_nobatch_int8", np.int8, [0]), ("identity_nobatch_int8", np.int8, [7]), ("identity_bytes", object, [1, 1]), - ("identity_bf16", ml_dtypes.bfloat16, [1, 0]), - ("identity_bf16", ml_dtypes.bfloat16, [1, 5]) + ("identity_bf16", np.float32, [1, 0]), + ("identity_bf16", np.float32, [1, 5]) ): # yapf: enable if np_dtype != object: @@ -235,20 +234,40 @@ print("error: expected 'OUTPUT0'") sys.exit(1) - if output_data.dtype != input_data.dtype: - print( - "error: expected output dtype {} to match input dtype {} for {}".format( - output_data.dtype, input_data.dtype, model_name + if model_name == "identity_bf16": + if input_data.shape != output_data.shape: + print( + "error: expected output shape {} to match input shape {}".format( + output_data.shape, input_data.shape + ) ) - ) - sys.exit(1) - if not np.array_equal(output_data, input_data): - print( - "error: expected output {} to match input {} for {}".format( - output_data, input_data, model_name + sys.exit(1) + for input, output in zip( + np.nditer(input_data, flags=["refs_ok", "zerosize_ok"], order="C"), + np.nditer(output_data, flags=["refs_ok", "zerosize_ok"], order="C"), + ): + if input.tobytes()[2:4] != output.tobytes()[2:4]: + print( + "error: expected low-order bits of output {} to match low-order bits of input {}".format( + output, input + ) + ) + sys.exit(1) + if output.tobytes()[0:2] != b"\x00\x00": + print( + "error: expected output {} to have all-zero high-order bits, got {}".format( + output, output.tobytes()[0:2] + ) + ) + sys.exit(1) + else: + if not np.array_equal(output_data, input_data): + print( + "error: expected output {} to match input {}".format( + output_data, input_data + ) ) - ) - sys.exit(1) + sys.exit(1) # Make sure response parameters are correct response = results.get_response() diff --git a/qa/L0_backend_onnxruntime/test.py b/qa/L0_backend_onnxruntime/test.py deleted file mode 100755 index 7575f6d0f5..0000000000 --- a/qa/L0_backend_onnxruntime/test.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import os -import unittest - -import ml_dtypes -import numpy as np -import tritonclient.grpc as grpcclient -import tritonclient.http as httpclient - - -class BFloat16Test(unittest.TestCase): - def setUp(self): - self.protocol = os.environ.get("CLIENT_TYPE", "http") - if self.protocol == "http": - self.client_ = httpclient.InferenceServerClient("localhost:8000") - self.client_module_ = httpclient - else: - self.client_ = grpcclient.InferenceServerClient("localhost:8001") - self.client_module_ = grpcclient - self.model_name_ = "onnx_bf16_bf16_bf16" - # Model dims are [-1, 16]: dynamic batch dim plus inner dim of 16. - self.inner_dim_ = 16 - - def _infer_bf16(self, input0_data, input1_data): - """Helper to run BF16 inference and return the output numpy arrays.""" - input0 = self.client_module_.InferInput( - "INPUT0", list(input0_data.shape), "BF16" - ) - input1 = self.client_module_.InferInput( - "INPUT1", list(input1_data.shape), "BF16" - ) - input0.set_data_from_numpy(input0_data) - input1.set_data_from_numpy(input1_data) - - results = self.client_.infer(self.model_name_, [input0, input1]) - return results.as_numpy("OUTPUT0"), results.as_numpy("OUTPUT1") - - def test_bf16_add_sub_variants(self): - """Run BF16 add/sub across multiple cases batched in a single request: - zeros, negatives, large, small, cancellation, and identical.""" - cases = [ - (0.0, 0.0), - (-1.5, 3.5), - (100.0, 200.0), - (1e-2, 1e-2), - (1.0, -1.0), - (2.0, 2.0), - ] - batch_size = len(cases) - input0_data = np.empty((batch_size, self.inner_dim_), dtype=ml_dtypes.bfloat16) - input1_data = np.empty((batch_size, self.inner_dim_), dtype=ml_dtypes.bfloat16) - for i, (v0, v1) in enumerate(cases): - input0_data[i, :] = v0 - input1_data[i, :] = v1 - - output0, output1 = self._infer_bf16(input0_data, input1_data) - self.assertEqual(output0.dtype, ml_dtypes.bfloat16) - self.assertEqual(output1.dtype, ml_dtypes.bfloat16) - np.testing.assert_array_equal(output0, input0_data + input1_data) - np.testing.assert_array_equal(output1, input0_data - input1_data) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_backend_python/common.sh b/qa/L0_backend_python/common.sh index 8b447b1713..70fb3cc081 100755 --- a/qa/L0_backend_python/common.sh +++ b/qa/L0_backend_python/common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -51,7 +51,7 @@ install_build_deps_apt() { && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ - && apt-get install -y --no-install-recommends cmake=4.0.3* cmake-data=4.0.3* + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* } install_build_deps_yum() { @@ -59,7 +59,7 @@ install_build_deps_yum() { } install_build_deps() { - if [[ ${TRITON_RHEL} -eq "1" ]] && grep -qE 'rhel|centos|fedora' /etc/os-release; then + if [[ ${TRITON_RHEL} -eq "1" ]]; then install_build_deps_yum else install_build_deps_apt @@ -85,22 +85,7 @@ create_conda_env_with_specified_path() { create_python_backend_stub() { rm -rf python_backend git clone ${TRITON_REPO_ORGANIZATION}/python_backend -b $PYTHON_BACKEND_REPO_TAG - CUDA_PATH=$(readlink -f /usr/local/cuda) - export CMAKE_POLICY_VERSION_MINIMUM=3.5 - (cd python_backend/ \ - && mkdir builddir \ - && cd builddir \ - && export CMAKE_POLICY_VERSION_MINIMUM=3.5 \ - && cmake \ - -DCMAKE_CUDA_COMPILER=$CUDA_PATH/bin/nvcc \ - -DCMAKE_INCLUDE_PATH:STRING=/usr/include \ - -DCUDAToolkit_ROOT=$CUDA_PATH \ - -DPYBIND11_PYTHON_VERSION=$PY_VERSION \ - -DTRITON_BACKEND_REPO_TAG=$TRITON_BACKEND_REPO_TAG \ - -DTRITON_COMMON_REPO_TAG=$TRITON_COMMON_REPO_TAG \ - -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG \ - -DTRITON_ENABLE_GPU=ON \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -S ../ \ - && cmake --build . --target triton-python-backend-stub -j18) + (cd python_backend/ && mkdir builddir && cd builddir && \ + cmake -DTRITON_ENABLE_GPU=ON -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_BACKEND_REPO_TAG=$TRITON_BACKEND_REPO_TAG -DTRITON_COMMON_REPO_TAG=$TRITON_COMMON_REPO_TAG -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG -DPYBIND11_PYTHON_VERSION=$PY_VERSION ../ && \ + make -j18 triton-python-backend-stub) } diff --git a/qa/L0_backend_python/decoupled/decoupled_test.py b/qa/L0_backend_python/decoupled/decoupled_test.py index d3442e03a0..45ce370fb1 100755 --- a/qa/L0_backend_python/decoupled/decoupled_test.py +++ b/qa/L0_backend_python/decoupled/decoupled_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -46,34 +46,6 @@ _tritonserver_ipaddr = os.environ.get("TRITONSERVER_IPADDR", "localhost") -def prepare_decoupled_bls_cancel_inputs(input_value, max_sum_value, ignore_cancel): - input_data = np.array([input_value], dtype=np.int32) - max_sum_data = np.array([max_sum_value], dtype=np.int32) - ignore_cancel_data = np.array([ignore_cancel], dtype=np.bool_) - inputs = [ - grpcclient.InferInput( - "INPUT", - input_data.shape, - np_to_triton_dtype(input_data.dtype), - ), - grpcclient.InferInput( - "MAX_SUM", - max_sum_data.shape, - np_to_triton_dtype(max_sum_data.dtype), - ), - grpcclient.InferInput( - "IGNORE_CANCEL", - ignore_cancel_data.shape, - np_to_triton_dtype(ignore_cancel_data.dtype), - ), - ] - inputs[0].set_data_from_numpy(input_data) - inputs[1].set_data_from_numpy(max_sum_data) - inputs[2].set_data_from_numpy(ignore_cancel_data) - - return inputs - - class UserData: def __init__(self): self._completed_requests = queue.Queue() @@ -352,171 +324,6 @@ def test_decoupled_execute_cancel(self): self.assertIn("[execute_cancel] Request not cancelled at 1.0 s", log_text) self.assertIn("[execute_cancel] Request cancelled at ", log_text) - def test_decoupled_bls_cancel(self): - model_names = ["decoupled_bls_cancel", "decoupled_bls_async_cancel"] - input_value = 1 - max_sum_value = 10 - ignore_cancel = False - user_data = UserData() - for model_name in model_names: - with self._shm_leak_detector.Probe() as shm_probe: - with grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) as client: - client.start_stream(callback=partial(callback, user_data)) - inputs = prepare_decoupled_bls_cancel_inputs( - input_value=input_value, - max_sum_value=max_sum_value, - ignore_cancel=ignore_cancel, - ) - client.async_stream_infer(model_name, inputs) - - # Check the results of the decoupled model using BLS - def check_result(result): - # Make sure the result is not an exception - self.assertIsNot(type(result), InferenceServerException) - is_cancelled = result.as_numpy("IS_CANCELLED") - self.assertTrue( - is_cancelled[0], - "error: expected the request to be cancelled", - ) - - max_sum_data = np.array([max_sum_value], dtype=np.int32) - sum_data = result.as_numpy("SUM") - self.assertIsNotNone(sum_data, "error: expected 'SUM'") - self.assertTrue( - np.array_equal(sum_data, max_sum_data), - "error: expected output {} to match input {}".format( - sum_data, max_sum_data - ), - ) - - result = user_data._completed_requests.get() - check_result(result) - - def test_decoupled_bls_ignore_cancel(self): - model_names = ["decoupled_bls_cancel", "decoupled_bls_async_cancel"] - input_value = 1 - max_sum_value = 10 - ignore_cancel = True - user_data = UserData() - for model_name in model_names: - with self._shm_leak_detector.Probe() as shm_probe: - with grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) as client: - client.start_stream(callback=partial(callback, user_data)) - inputs = prepare_decoupled_bls_cancel_inputs( - input_value=input_value, - max_sum_value=max_sum_value, - ignore_cancel=ignore_cancel, - ) - client.async_stream_infer(model_name, inputs) - - # Check the results of the decoupled model using BLS - def check_result(result): - # Make sure the result is not an exception - self.assertIsNot(type(result), InferenceServerException) - is_cancelled = result.as_numpy("IS_CANCELLED") - self.assertFalse( - is_cancelled[0], - "error: expected the request not being cancelled", - ) - - max_sum_data = np.array([max_sum_value], dtype=np.int32) - sum_data = result.as_numpy("SUM") - self.assertIsNotNone(sum_data, "error: expected 'SUM'") - self.assertTrue( - sum_data > max_sum_data, - "error: expected sum_data {} to be greater than max_sum_data {}".format( - sum_data, max_sum_data - ), - ) - - result = user_data._completed_requests.get() - check_result(result) - - def test_decoupled_bls_cancel_after_cancellation(self): - model_name = "decoupled_bls_cancel_after_complete" - input_value = 1 - max_sum_value = 10 - ignore_cancel = False - user_data = UserData() - with self._shm_leak_detector.Probe() as shm_probe: - with grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) as client: - client.start_stream(callback=partial(callback, user_data)) - inputs = prepare_decoupled_bls_cancel_inputs( - input_value=input_value, - max_sum_value=max_sum_value, - ignore_cancel=ignore_cancel, - ) - client.async_stream_infer(model_name, inputs) - - # Check the results of the decoupled model using BLS - def check_result(result): - # Make sure the result is not an exception - self.assertIsNot(type(result), InferenceServerException) - is_cancelled = result.as_numpy("IS_CANCELLED") - self.assertTrue( - is_cancelled[0], "error: expected the request to be cancelled" - ) - - max_sum_data = np.array([max_sum_value], dtype=np.int32) - sum_data = result.as_numpy("SUM") - self.assertIsNotNone(sum_data, "error: expected 'SUM'") - self.assertTrue( - np.array_equal(sum_data, max_sum_data), - "error: expected output {} to match input {}".format( - sum_data, max_sum_data - ), - ) - - result = user_data._completed_requests.get() - check_result(result) - - def test_decoupled_bls_cancel_after_completion(self): - model_name = "decoupled_bls_cancel_after_complete" - input_value = 1 - max_sum_value = 25 - ignore_cancel = False - user_data = UserData() - with self._shm_leak_detector.Probe() as shm_probe: - with grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) as client: - client.start_stream(callback=partial(callback, user_data)) - inputs = prepare_decoupled_bls_cancel_inputs( - input_value=input_value, - max_sum_value=max_sum_value, - ignore_cancel=ignore_cancel, - ) - client.async_stream_infer(model_name, inputs) - - # Check the results of the decoupled model using BLS - def check_result(result): - # Make sure the result is not an exception - self.assertIsNot(type(result), InferenceServerException) - is_cancelled = result.as_numpy("IS_CANCELLED") - self.assertFalse( - is_cancelled[0], - "error: expected the request not being cancelled", - ) - - max_sum_data = np.array([max_sum_value], dtype=np.int32) - sum_data = result.as_numpy("SUM") - self.assertIsNotNone(sum_data, "error: expected 'SUM'") - self.assertTrue( - sum_data < max_sum_data, - "error: expected sum_data {} to be lesser than max_sum_data {}".format( - sum_data, max_sum_data - ), - ) - - result = user_data._completed_requests.get() - check_result(result) - def test_decoupled_raise_exception(self): # The decoupled_raise_exception model raises an exception for the request. # This test case is making sure that repeated exceptions are properly handled. diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/1/model.py b/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/1/model.py deleted file mode 100644 index d84221e7e2..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/1/model.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import asyncio - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - This model sends a decoupled bls inference request to 'response_sender_until_cancelled' - model, and sums up its responses. - Once the MAX_SUM is reached, the model will call the response iterator's - cancel() method to cancel the response stream. - If the IGNORE_CANCEL is set to True, the 'response_sender_until_cancelled' model will not hornor - the request cancellation and keep sending the output to the model. - The number of total responses should not reach MAX_RESPONSE_COUNT. - """ - - async def execute(self, requests): - max_sum = ( - pb_utils.get_input_tensor_by_name(requests[0], "MAX_SUM").as_numpy().flat[0] - ) - input = pb_utils.get_input_tensor_by_name(requests[0], "INPUT") - ignore_cancel = pb_utils.get_input_tensor_by_name(requests[0], "IGNORE_CANCEL") - delay = pb_utils.Tensor("DELAY", np.array([50], dtype=np.int32)) - max_response_count = pb_utils.Tensor( - "MAX_RESPONSE_COUNT", np.array([20], dtype=np.int32) - ) - - infer_request = pb_utils.InferenceRequest( - model_name="response_sender_until_cancelled", - inputs=[input, max_response_count, delay, ignore_cancel], - requested_output_names=["OUTPUT"], - ) - - response_stream = await infer_request.async_exec(decoupled=True) - - is_cancelled = False - error = None - response_sum = 0 - for infer_response in response_stream: - if infer_response.has_error(): - if infer_response.error().code() == pb_utils.TritonError.CANCELLED: - is_cancelled = True - else: - error = infer_response.error() - break - - out = pb_utils.get_output_tensor_by_name( - infer_response, "OUTPUT" - ).as_numpy()[0] - - response_sum += out - if response_sum >= max_sum: - response_stream.cancel() - - responses = [ - pb_utils.InferenceResponse( - output_tensors=[ - pb_utils.Tensor("SUM", np.array([response_sum], dtype=np.int32)), - pb_utils.Tensor( - "IS_CANCELLED", np.array([is_cancelled], dtype=np.bool_) - ), - ], - error=error, - ) - ] - - return responses diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/config.pbtxt b/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/config.pbtxt deleted file mode 100644 index 83cc39a6f6..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_async_cancel/config.pbtxt +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "decoupled_bls_async_cancel" -backend: "python" - -input [ - { - name: "INPUT" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "MAX_SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IGNORE_CANCEL" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] -output [ - { - name: "SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IS_CANCELLED" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] - -instance_group [ - { - count: 1 - kind : KIND_CPU - } -] diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/1/model.py b/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/1/model.py deleted file mode 100644 index dab6414ede..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/1/model.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - This model sends a decoupled bls inference request to 'response_sender_until_cancelled' - model, and sums up its responses. - Once the MAX_SUM is reached, the model will call the response iterator's - cancel() method to cancel the response stream. - If the IGNORE_CANCEL is set to True, the 'response_sender_until_cancelled' model will not hornor - the request cancellation and keep sending the output to the model. - The number of total responses should not reach MAX_RESPONSE_COUNT. - """ - - def execute(self, requests): - max_sum = ( - pb_utils.get_input_tensor_by_name(requests[0], "MAX_SUM").as_numpy().flat[0] - ) - input = pb_utils.get_input_tensor_by_name(requests[0], "INPUT") - ignore_cancel = pb_utils.get_input_tensor_by_name(requests[0], "IGNORE_CANCEL") - delay = pb_utils.Tensor("DELAY", np.array([50], dtype=np.int32)) - max_response_count = pb_utils.Tensor( - "MAX_RESPONSE_COUNT", np.array([20], dtype=np.int32) - ) - - infer_request = pb_utils.InferenceRequest( - model_name="response_sender_until_cancelled", - inputs=[input, max_response_count, delay, ignore_cancel], - requested_output_names=["OUTPUT"], - ) - - response_stream = infer_request.exec(decoupled=True) - - is_cancelled = False - error = None - response_sum = 0 - for infer_response in response_stream: - if infer_response.has_error(): - if infer_response.error().code() == pb_utils.TritonError.CANCELLED: - is_cancelled = True - else: - error = infer_response.error() - break - - out = pb_utils.get_output_tensor_by_name( - infer_response, "OUTPUT" - ).as_numpy()[0] - - response_sum += out - if response_sum >= max_sum: - response_stream.cancel() - - responses = [ - pb_utils.InferenceResponse( - output_tensors=[ - pb_utils.Tensor("SUM", np.array([response_sum], dtype=np.int32)), - pb_utils.Tensor( - "IS_CANCELLED", np.array([is_cancelled], dtype=np.bool_) - ), - ], - error=error, - ) - ] - - return responses diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/config.pbtxt b/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/config.pbtxt deleted file mode 100644 index 0b443de6f9..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel/config.pbtxt +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "decoupled_bls_cancel" -backend: "python" - -input [ - { - name: "INPUT" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "MAX_SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IGNORE_CANCEL" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] -output [ - { - name: "SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IS_CANCELLED" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] - -instance_group [ - { - count: 1 - kind : KIND_CPU - } -] diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/1/model.py b/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/1/model.py deleted file mode 100644 index 4c4202de1a..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/1/model.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import asyncio - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - This model sends a decoupled bls inference request to 'response_sender_until_cancelled' - model, and sums up its responses. - Once the MAX_SUM is reached, the model will call the response iterator's - cancel() method to cancel the response stream. - If the IGNORE_CANCEL is set to True, the 'response_sender_until_cancelled' model will not hornor - the request cancellation and keep sending the output to the model. - The number of total responses should not reach MAX_RESPONSE_COUNT. - """ - - async def execute(self, requests): - max_sum = ( - pb_utils.get_input_tensor_by_name(requests[0], "MAX_SUM").as_numpy().flat[0] - ) - input = pb_utils.get_input_tensor_by_name(requests[0], "INPUT") - ignore_cancel = pb_utils.get_input_tensor_by_name(requests[0], "IGNORE_CANCEL") - delay = pb_utils.Tensor("DELAY", np.array([50], dtype=np.int32)) - max_response_count = pb_utils.Tensor( - "MAX_RESPONSE_COUNT", np.array([20], dtype=np.int32) - ) - - infer_request = pb_utils.InferenceRequest( - model_name="response_sender_until_cancelled", - inputs=[input, max_response_count, delay, ignore_cancel], - requested_output_names=["OUTPUT"], - ) - - response_stream = await infer_request.async_exec(decoupled=True) - - is_cancelled = False - error = None - response_sum = 0 - for infer_response in response_stream: - if infer_response.has_error(): - if infer_response.error().code() == pb_utils.TritonError.CANCELLED: - is_cancelled = True - else: - error = infer_response.error() - break - - out = pb_utils.get_output_tensor_by_name( - infer_response, "OUTPUT" - ).as_numpy()[0] - - response_sum += out - if response_sum >= max_sum: - response_stream.cancel() - - # test cancel after request completion. - if not error: - try: - response_stream.cancel() - except Exception as e: - error = pb_utils.TritonError( - message=str(e), - code=pb_utils.TritonError.INTERNAL, - ) - - responses = [ - pb_utils.InferenceResponse( - output_tensors=[ - pb_utils.Tensor("SUM", np.array([response_sum], dtype=np.int32)), - pb_utils.Tensor( - "IS_CANCELLED", np.array([is_cancelled], dtype=np.bool_) - ), - ], - error=error, - ) - ] - - return responses diff --git a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/config.pbtxt b/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/config.pbtxt deleted file mode 100644 index cdbffa5419..0000000000 --- a/qa/L0_backend_python/decoupled/models/decoupled_bls_cancel_after_complete/config.pbtxt +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "decoupled_bls_cancel_after_complete" -backend: "python" - -input [ - { - name: "INPUT" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "MAX_SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IGNORE_CANCEL" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] -output [ - { - name: "SUM" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IS_CANCELLED" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] - -instance_group [ - { - count: 1 - kind : KIND_CPU - } -] diff --git a/qa/L0_backend_python/decoupled/test.sh b/qa/L0_backend_python/decoupled/test.sh index 622c151050..672335c892 100755 --- a/qa/L0_backend_python/decoupled/test.sh +++ b/qa/L0_backend_python/decoupled/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,10 +53,6 @@ cp ../../python_models/execute_cancel/model.py ./models/execute_cancel/1/ cp ../../python_models/execute_cancel/config.pbtxt ./models/execute_cancel/ echo "model_transaction_policy { decoupled: True }" >> ./models/execute_cancel/config.pbtxt -mkdir -p models/response_sender_until_cancelled/1/ -cp ../../python_models/response_sender_until_cancelled/model.py ./models/response_sender_until_cancelled/1/ -cp ../../python_models/response_sender_until_cancelled/config.pbtxt ./models/response_sender_until_cancelled/ - rm -fr python_backend git clone ${TRITON_REPO_ORGANIZATION}/python_backend -b $PYTHON_BACKEND_REPO_TAG mkdir -p models/square_int32/1/ diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh index b98ba607d6..4d9de30a56 100755 --- a/qa/L0_backend_python/env/test.sh +++ b/qa/L0_backend_python/env/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,13 +48,13 @@ install_conda path_to_conda_pack='$$TRITON_MODEL_DIRECTORY/python_3_12_environment.tar.gz' create_conda_env "3.12" "python-3-12" conda install -c conda-forge libstdcxx-ng=14 -y -TORCH_VERSION="2.8.0" +TORCH_VERSION="2.6.0" conda install numpy=1.26.4 -y -if [[ ${TRITON_RHEL} -eq "1" ]] && grep -qE 'rhel|centos|fedora' /etc/os-release; then +if [ $TRITON_RHEL -eq 1 ]; then TORCH_VERISON="2.17.0" fi -conda install pytorch=${TORCH_VERSION} -y -PY312_VERSION_STRING="Python version is 3.12, NumPy version is 1.26.4, and PyTorch version is ${TORCH_VERSION}" +conda install torch=${TORCH_VERSION} -y +PY312_VERSION_STRING="Python version is 3.12, NumPy version is 1.26.4, and PyTorch version is ${TORCH_VERISON}" conda pack -o python3.12.tar.gz mkdir -p models/python_3_12/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_12 @@ -122,7 +122,7 @@ fi kill_server set +e -grep "Locale is ('C', 'UTF-8')" $SERVER_LOG +grep "Locale is ('en_US', 'UTF-8')" $SERVER_LOG if [ $? -ne 0 ]; then cat $SERVER_LOG echo -e "\n***\n*** Locale UTF-8 was not found in Triton logs. \n***" @@ -182,6 +182,10 @@ aws s3 mb "${BUCKET_URL}" BUCKET_URL=${BUCKET_URL%/} BUCKET_URL_SLASH="${BUCKET_URL}/" +# Remove Python 3.7 model because it contains absolute paths and cannot be used +# with S3. +rm -rf models/python_3_7 + # Test with the bucket url as model repository aws s3 cp models/ "${BUCKET_URL_SLASH}" --recursive --include "*" @@ -201,10 +205,10 @@ fi kill_server set +e -grep "$PY312_VERSION_STRING" $SERVER_LOG +grep "$PY36_VERSION_STRING" $SERVER_LOG if [ $? -ne 0 ]; then cat $SERVER_LOG - echo -e "\n***\n*** $PY312_VERSION_STRING was not found in Triton logs. \n***" + echo -e "\n***\n*** $PY36_VERSION_STRING was not found in Triton logs. \n***" RET=1 fi set -e @@ -213,6 +217,8 @@ set -e aws s3 rm "${BUCKET_URL_SLASH}" --recursive --include "*" # Test with EXECUTION_ENV_PATH outside the model directory +sed -i "s/TRITON_MODEL_DIRECTORY\/python_3_6_environment/TRITON_MODEL_DIRECTORY\/..\/python_3_6_environment/" models/python_3_6/config.pbtxt +mv models/python_3_6/python_3_6_environment.tar.gz models sed -i "s/\$\$TRITON_MODEL_DIRECTORY\/python_3_12_environment/s3:\/\/triton-bucket-${CI_JOB_ID}\/python_3_12_environment/" models/python_3_12/config.pbtxt mv models/python_3_12/python_3_12_environment.tar.gz models @@ -232,7 +238,7 @@ fi kill_server set +e -for EXPECTED_VERSION_STRING in "$PY312_VERSION_STRING"; do +for EXPECTED_VERSION_STRING in "$PY36_VERSION_STRING" "$PY312_VERSION_STRING"; do grep "$EXPECTED_VERSION_STRING" $SERVER_LOG if [ $? -ne 0 ]; then cat $SERVER_LOG @@ -246,63 +252,6 @@ set -e aws s3 rm "${BUCKET_URL_SLASH}" --recursive --include "*" aws s3 rb "${BUCKET_URL}" -# EXECUTION_ENV_PATH path-traversal regression test. For each -# traversal vector, load a Python-backend model with archive is malicious -# and assert: (a) Triton refuses to start, (b) no file was written outside -# the extraction directory, (c) the server log contains the libarchive -# rejection. See zipslip_test.py for the archive layout. -ZIPSLIP_REPO="`pwd`/zipslip_models" -ZIPSLIP_EXECUTION_ENV_PATH='$$TRITON_MODEL_DIRECTORY/malicious_env.tar.gz' - -ZIPSLIP_CASES=( - "relative:Path contains '..'" - "absolute:Path is absolute" -) - -for case in "${ZIPSLIP_CASES[@]}"; do - mode="${case%%:*}" - expected_msg="${case#*:}" - model="zipslip_${mode}" - marker="/tmp/zipslip_${mode}_marker_$$" - SERVER_LOG="./zipslip_${mode}_server.log" - - rm -rf "${marker}" "${SERVER_LOG}" "${ZIPSLIP_REPO}" - mkdir -p "${ZIPSLIP_REPO}/${model}/1" - - cp ../../python_models/identity_fp32/config.pbtxt "${ZIPSLIP_REPO}/${model}/config.pbtxt" - cp ../../python_models/identity_fp32/model.py "${ZIPSLIP_REPO}/${model}/1/model.py" - (cd "${ZIPSLIP_REPO}/${model}" && \ - sed -i "s/^name:.*/name: \"${model}\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$ZIPSLIP_EXECUTION_ENV_PATH\"}}" >> config.pbtxt) - python3 ./zipslip_test.py \ - --mode "${mode}" \ - --output "${ZIPSLIP_REPO}/${model}/malicious_env.tar.gz" \ - --marker "${marker}" - - SERVER_ARGS="--model-repository=${ZIPSLIP_REPO} --log-verbose=1" - run_server - if [ "$SERVER_PID" != "0" ]; then - kill_server - echo -e "\n***\n*** Zip Slip (mode=${mode}): tritonserver started despite malicious EXECUTION_ENV_PATH archive.\n***" - RET=1 - fi - - set +e - if [ -e "${marker}" ]; then - ls -la "${marker}" - echo -e "\n***\n*** Zip Slip (mode=${mode}): marker file written outside extraction directory.\n***" - RET=1 - fi - if ! grep -q "${expected_msg}" "${SERVER_LOG}"; then - cat "${SERVER_LOG}" - echo -e "\n***\n*** Zip Slip (mode=${mode}): expected '${expected_msg}' not found in server log.\n***" - RET=1 - fi - set -e - - rm -f "${marker}" -done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Env Manager Test PASSED.\n***" else diff --git a/qa/L0_backend_python/env/zipslip_test.py b/qa/L0_backend_python/env/zipslip_test.py deleted file mode 100755 index 83336a245f..0000000000 --- a/qa/L0_backend_python/env/zipslip_test.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" -Build a malicious EXECUTION_ENV_PATH tarball for the Python backend path-traversal regression test. - - --mode relative -> entry '../../' (ARCHIVE_EXTRACT_SECURE_NODOTDOT) - --mode absolute -> entry '/tmp/' (ARCHIVE_EXTRACT_SECURE_NOABSOLUTEPATHS) -""" - -import argparse -import io -import os -import sys -import tarfile -import time - - -def build_archive(mode: str, output: str, marker: str) -> None: - if mode == "relative": - entry_name = "../../" + marker[len("/tmp/") :] - else: - entry_name = marker - - def add(tar: tarfile.TarFile, name: str, body: bytes, mode: int = 0o644): - info = tarfile.TarInfo(name=name) - info.size, info.mode, info.mtime = len(body), mode, int(time.time()) - tar.addfile(info, io.BytesIO(body)) - - os.makedirs(os.path.dirname(output) or ".", exist_ok=True) - with tarfile.open(output, "w:gz") as tar: - add(tar, entry_name, b"traversal-test\n") - # Stand-in for a conda-pack activate script so an unpatched server - # accepts the model end-to-end rather than failing on the missing - # entry-point. - add(tar, "bin/activate", b"#!/bin/bash\n", mode=0o755) - - print(f"[zipslip] wrote {output} (mode={mode}, entry={entry_name!r})") - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - ap.add_argument("--mode", required=True, choices=["relative", "absolute"]) - ap.add_argument( - "--output", required=True, help="path to write the malicious .tar.gz" - ) - ap.add_argument( - "--marker", - required=True, - help="absolute path under /tmp/ where the traversal " - "would land on a vulnerable server", - ) - args = ap.parse_args() - - try: - build_archive(args.mode, args.output, args.marker) - except ValueError as e: - print(f"error: {e}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/qa/L0_backend_python/examples/test.sh b/qa/L0_backend_python/examples/test.sh index c074b8e310..ea0fcd992c 100755 --- a/qa/L0_backend_python/examples/test.sh +++ b/qa/L0_backend_python/examples/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -69,7 +69,7 @@ pip3 install validators # Install JAX # Jax has dropped the support for Python 3.8. See https://jax.readthedocs.io/en/latest/changelog.html if [ "$TEST_JETSON" == "0" ] && [ ${PYTHON_ENV_VERSION} != "8" ]; then - pip install -U "jax[cuda12]" + pip3 install --upgrade "jax[cuda12_local]" -f https://storage.googleapis.com/jax-releases/jax_cuda_releases.html fi git clone ${TRITON_REPO_ORGANIZATION}/python_backend -b $PYTHON_BACKEND_REPO_TAG diff --git a/qa/L0_backend_python/lifecycle/lifecycle_test.py b/qa/L0_backend_python/lifecycle/lifecycle_test.py index e16c0944ff..3874ef428e 100755 --- a/qa/L0_backend_python/lifecycle/lifecycle_test.py +++ b/qa/L0_backend_python/lifecycle/lifecycle_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -255,7 +255,7 @@ def test_triton_grpc_error_error_on(self): callback=partial(callback, user_data), headers=metadata ) stream_end = False - for _ in range(number_of_requests): + for i in range(number_of_requests): input_data = np.random.randn(*shape).astype(np.float32) inputs = [ grpcclient.InferInput( @@ -348,68 +348,6 @@ def test_triton_grpc_error_cancel(self): "This should always pass as cancellation should succeed without any exception", ) - # Regression test for a segfault when a decoupled model sends an error - # response followed by a delayed FINAL flag with "triton_grpc_error" enabled. - # The error triggers stream closure, and the late FINAL callback must not - # crash by accessing a released state. - def test_triton_grpc_error_decoupled_delayed_final(self): - model_name = "decoupled_grpc_error" - shape = [2] - user_data = UserData() - metadata = {"triton_grpc_error": "true"} - - for mode in [ - "MODE_ERROR_ONLY", - "MODE_ERROR_FINAL", - "MODE_ERROR_WITH_DELAYED_FINAL", - ]: - triton_client = grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) - triton_client.start_stream( - callback=partial(callback, user_data), headers=metadata - ) - - input_data = np.random.randn(*shape).astype(np.float32) - mode_data = np.array([bytes(mode, "utf-8")], dtype=object) - inputs = [ - grpcclient.InferInput( - "IN", input_data.shape, np_to_triton_dtype(input_data.dtype) - ), - grpcclient.InferInput("MODE", mode_data.shape, "BYTES"), - ] - inputs[0].set_data_from_numpy(input_data) - inputs[1].set_data_from_numpy(mode_data) - triton_client.async_stream_infer(model_name=model_name, inputs=inputs) - - # The first response is always identical to the input data. - result = user_data._completed_requests.get() - output_data = result.as_numpy("OUT") - self.assertIsNotNone(output_data, "error: expected 'OUT'") - self.assertTrue( - np.array_equal(output_data, input_data), - "error: expected output {} to match input {}".format( - output_data, input_data - ), - ) - - # The second response is the error response. - result = user_data._completed_requests.get() - self.assertIsInstance(result, InferenceServerException) - self.assertEqual(str(result.status()), "StatusCode.INTERNAL") - - # Should not receive any subsequent responses. - with self.assertRaises(queue.Empty): - # Wait for the delayed FINAL flag (model sleeps 0.5s) plus buffer. - # Before the fix, the server would SIGSEGV here. - user_data._completed_requests.get(timeout=2) - - # Verify the server is still alive after the delayed FINAL flag. - triton_client2 = grpcclient.InferenceServerClient( - f"{_tritonserver_ipaddr}:8001" - ) - self.assertTrue(triton_client2.is_server_live()) - # Test grpc stream behavior when triton_grpc_error is set to false # and subsequent stream is NOT closed when error is reported from CORE def test_triton_grpc_error_error_off(self): diff --git a/qa/L0_backend_python/lifecycle/test.sh b/qa/L0_backend_python/lifecycle/test.sh index f53c219772..59b846f56b 100755 --- a/qa/L0_backend_python/lifecycle/test.sh +++ b/qa/L0_backend_python/lifecycle/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -60,10 +60,6 @@ cp ../../python_models/execute_grpc_error/config.pbtxt ./models/execute_grpc_err sed -i "s/^max_batch_size:.*/max_batch_size: 8/" config.pbtxt && \ echo "dynamic_batching { preferred_batch_size: [8], max_queue_delay_microseconds: 1200000 }" >> config.pbtxt) -mkdir -p models/decoupled_grpc_error/1/ -cp ../../python_models/decoupled_grpc_error/model.py ./models/decoupled_grpc_error/1/ -cp ../../python_models/decoupled_grpc_error/config.pbtxt ./models/decoupled_grpc_error/ - mkdir -p models/execute_return_error/1/ cp ../../python_models/execute_return_error/model.py ./models/execute_return_error/1/ cp ../../python_models/execute_return_error/config.pbtxt ./models/execute_return_error/ diff --git a/qa/L0_backend_python/model_control/model_control_test.py b/qa/L0_backend_python/model_control/model_control_test.py index a0ad197bb8..9ccb73df4f 100755 --- a/qa/L0_backend_python/model_control/model_control_test.py +++ b/qa/L0_backend_python/model_control/model_control_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -26,10 +26,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import base64 -import json import os -import subprocess import sys sys.path.append("../../common") @@ -86,241 +83,5 @@ def test_model_reload(self): self.assertFalse(client.is_model_ready(ensemble_model_name)) -class ModelIDValidationTest(unittest.TestCase): - """ - Test model ID validation for user-provided model names. - - Verifies that model names containing dangerous characters are properly rejected. - Uses raw HTTP requests via curl instead of the Triton client to test server-side - validation without the Triton client encoding special characters. - """ - - def setUp(self): - self._shm_leak_detector = shm_util.ShmLeakDetector() - self._client = httpclient.InferenceServerClient(f"{_tritonserver_ipaddr}:8000") - self._triton_host = _tritonserver_ipaddr - self._triton_port = 8000 - - # Check if curl is available - try: - subprocess.run(["curl", "--version"], capture_output=True, check=True) - except (subprocess.CalledProcessError, FileNotFoundError): - self.skipTest("curl command not available - required for raw HTTP testing") - - def _send_load_model_request(self, model_name): - """Send HTTP request to load model for testing input validation using curl""" - - # Create simple Triton Python model code - python_model_code = f"""import triton_python_backend_utils as pb_utils - -class TritonPythonModel: - def execute(self, requests): - print('Hello world from model {model_name}') - responses = [] - for request in requests: - # Simple identity function - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses""" - - # Base64 encode the Python code (as required by Triton server) - python_code_b64 = base64.b64encode(python_model_code.encode("utf-8")).decode( - "ascii" - ) - - # Create simple config - config = { - "name": model_name, - "backend": "python", - "max_batch_size": 4, - "input": [{"name": "INPUT0", "data_type": "TYPE_FP32", "dims": [-1]}], - "output": [{"name": "OUTPUT0", "data_type": "TYPE_FP32", "dims": [-1]}], - } - - payload = { - "parameters": { - "config": json.dumps(config), - "file:/1/model.py": python_code_b64, - } - } - - url = f"http://{self._triton_host}:{self._triton_port}/v2/repository/models/{model_name}/load" - - # Convert payload to JSON string - payload_json = json.dumps(payload) - - try: - # Use curl to send the request - curl_cmd = [ - "curl", - "-s", - "-w", - "\n%{http_code}", # Write HTTP status code on separate line - "-X", - "POST", - "-H", - "Content-Type: application/json", - "-d", - payload_json, - url, - ] - - result = subprocess.run( - curl_cmd, capture_output=True, text=True, timeout=10 - ) - - # Parse curl output - last line is status code, rest is response body - output_lines = ( - result.stdout.strip().split("\n") if result.stdout.strip() else [] - ) - if len(output_lines) >= 2: - try: - status_code = int(output_lines[-1]) - response_text = "\n".join(output_lines[:-1]) - except ValueError: - status_code = 0 - response_text = result.stdout or result.stderr or "Invalid response" - elif len(output_lines) == 1 and output_lines[0].isdigit(): - status_code = int(output_lines[0]) - response_text = result.stderr or "No response body" - else: - status_code = 0 - response_text = result.stdout or result.stderr or "No response" - - # Return an object similar to requests.Response - class CurlResponse: - def __init__(self, status_code, text): - self.status_code = status_code - self.text = text - self.content = text.encode() - - return CurlResponse(status_code, response_text) - - except ( - subprocess.TimeoutExpired, - subprocess.CalledProcessError, - ValueError, - ) as e: - # Return a mock response for errors - class ErrorResponse: - def __init__(self, error_msg): - self.status_code = 0 - self.text = f"Error: {error_msg}" - self.content = self.text.encode() - - return ErrorResponse(str(e)) - - def test_invalid_character_model_names(self): - """Test that model names with invalid characters are properly rejected""" - - # Based on INVALID_CHARS = ";|&$`<>()[]{}\\\"'*?~#!" - invalid_model_names = [ - r"model;test", - r"model|test", - r"model&test", - r"model$test", - r"model`test`", - r"model", - r"model(test)", - # r"model[test]", # request fails to send unencoded - r"model{test}", - r"model\test", - r'model"test"', - r"model'test'", - r"model*test", - # r"model?test", # request fails to send unencoded - r"model~test", - # r"model#test", # request fails to send unencoded - r"model!test", - ] - - for invalid_name in invalid_model_names: - with self.subTest(model_name=invalid_name): - print(f"Testing invalid model name: {invalid_name}") - - response = self._send_load_model_request(invalid_name) - print( - f"Response for '{invalid_name}': Status {response.status_code}, Text: {response.text[:200]}..." - ) - - # Should not get a successful 200 response - self.assertNotEqual( - 200, - response.status_code, - f"Invalid model name '{invalid_name}' should not get 200 OK response", - ) - - # Special case for curly braces - they get stripped and cause load failures prior to the validation check - if "{" in invalid_name or "}" in invalid_name: - self.assertIn( - "failed to load", - response.text, - f"Model with curly braces '{invalid_name}' should fail to load", - ) - else: - # Normal case - should get character validation error - self.assertIn( - "Invalid stub name: contains invalid characters", - response.text, - f"invalid response for '{invalid_name}' should contain 'Invalid stub name: contains invalid characters'", - ) - - # Verify the model is not loaded/ready since it was rejected - try: - self.assertFalse( - self._client.is_model_ready(invalid_name), - f"Model '{invalid_name}' should not be ready after failed load attempt", - ) - except Exception as e: - # If checking model readiness fails, that's also acceptable since the model name is invalid - print( - f"Note: Could not check model readiness for '{invalid_name}': {e}" - ) - - def test_valid_model_names(self): - """Test that valid model names work""" - - valid_model_names = [ - "TestModel123", - "model-with-hyphens", - "model_with_underscores", - ] - - for valid_name in valid_model_names: - with self.subTest(model_name=valid_name): - print(f"Testing valid model name: {valid_name}") - - response = self._send_load_model_request(valid_name) - print( - f"Response for valid '{valid_name}': Status {response.status_code}, Text: {response.text[:100]}..." - ) - - # Valid model names should be accepted and load successfully - self.assertEqual( - 200, - response.status_code, - f"Valid model name '{valid_name}' should get 200 OK response, got {response.status_code}. Response: {response.text}", - ) - - # Should not contain validation error message - self.assertNotIn( - "Invalid stub name: contains invalid characters", - response.text, - f"Valid model name '{valid_name}' should not contain validation error message", - ) - - # Verify the model is actually loaded by checking if it's ready - try: - self.assertTrue( - self._client.is_model_ready(valid_name), - f"Model '{valid_name}' should be ready after successful load", - ) - # Clean up - unload the model after testing - self._client.unload_model(valid_name) - except Exception as e: - self.fail(f"Failed to check if model '{valid_name}' is ready: {e}") - - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_backend_python/model_control/test.sh b/qa/L0_backend_python/model_control/test.sh index f841222a1b..e2c22f2685 100755 --- a/qa/L0_backend_python/model_control/test.sh +++ b/qa/L0_backend_python/model_control/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,24 +55,11 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** model_control_test.py FAILED. \n***" RET=1 fi - -echo -e "\n***\n*** Running model ID validation test\n***" -SUBTEST="model_id_validation" -python3 -m pytest --junitxml=model_control.${SUBTEST}.report.xml model_control_test.py::ModelIDValidationTest >> ${CLIENT_LOG} 2>&1 - -if [ $? -ne 0 ]; then - echo -e "\n***\n*** model_id_validation_test.py FAILED. \n***" - RET=1 -fi - set -e kill_server if [ $RET -eq 1 ]; then - echo -e "\n***\n*** Server logs:\n***" - cat $SERVER_LOG - echo -e "\n***\n*** Client logs:\n***" cat $CLIENT_LOG echo -e "\n***\n*** model_control_test FAILED. \n***" else diff --git a/qa/L0_backend_python/model_readiness/test.sh b/qa/L0_backend_python/model_readiness/test.sh deleted file mode 100755 index d56dd29a25..0000000000 --- a/qa/L0_backend_python/model_readiness/test.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -TEST_RESULT_FILE='test_results.txt' -source ../common.sh -source ../../common/util.sh - -SERVER_ARGS="--model-repository=${MODELDIR}/model_readiness/models --backend-directory=${BACKEND_DIR} --log-verbose=1" - -RET=0 -rm -fr *.log ./models - -MODEL_NAME="identity_fp32" -mkdir -p models/$MODEL_NAME/1/ -cp ../../python_models/$MODEL_NAME/model.py ./models/$MODEL_NAME/1/model.py -cp ../../python_models/$MODEL_NAME/config.pbtxt ./models/$MODEL_NAME/config.pbtxt - -# -# Test Model Readiness (TRITONBACKEND_ModelInstanceReady) -# Test with different signals to simulate various crash/exit scenarios -# 11 (SIGSEGV) - Segmentation fault / crash -# 9 (SIGKILL) - Force kill -for SIGNAL in 11 9; do - echo -e "\n***\n*** Testing model_readiness with Signal $SIGNAL\n***" - SERVER_LOG="./model_readiness_signal_${SIGNAL}_server.log" - CLIENT_LOG="./model_readiness_signal_${SIGNAL}_client.log" - - run_server - if [ "$SERVER_PID" == "0" ]; then - cat $SERVER_LOG - echo -e "\n***\n*** Failed to start $SERVER\n***" - exit 1 - fi - - set +e - - # Verify model is initially ready - echo "Checking Initial Readiness..." - python3 -m unittest test_model_readiness.TestModelReadiness.test_model_ready >> ${CLIENT_LOG} 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test model_readiness Failed (Signal $SIGNAL): Initial readiness check failed \n***" - RET=1 - kill_server - exit 1 - fi - - # Find the stub process PID - stub_pid=$(pgrep -f "triton_python_backend_stub") - - if [ -z "$stub_pid" ]; then - echo -e "\n***\n*** Test model_readiness Failed (Signal $SIGNAL): Could not find stub process \n***" - RET=1 - kill_server - else - echo "Found stub process: $stub_pid" - - # Kill the stub process - echo "Killing stub with signal $SIGNAL..." - kill -$SIGNAL $stub_pid - sleep 1 - - # Verify model is now NOT ready - echo "Checking Not Ready Status..." - python3 -m unittest test_model_readiness.TestModelReadiness.test_model_not_ready >> ${CLIENT_LOG} 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test model_readiness Failed (Signal $SIGNAL): Model reported ready after kill \n***" - RET=1 - else - # Verify correct error message in logs - # Expect 2 occurrences: HTTP and gRPC checks - error_count=$(grep -c "Model '${MODEL_NAME}' version 1 is not ready: Stub process '${MODEL_NAME}_0_0' is not healthy." $SERVER_LOG) - if [ "$error_count" -eq 2 ]; then - echo -e "\n***\n Test model_readiness Passed for Signal $SIGNAL \n***" - else - echo -e "\n***\n*** Test model_readiness Failed (Signal $SIGNAL): Expected 2 error messages, found $error_count \n***" - cat $SERVER_LOG - RET=1 - fi - fi - fi - - set -e - kill_server -done - -# -# Test User-Defined Model Readiness Function -# -echo -e "\n***\n*** Testing User-Defined is_ready() Function\n***" - -# Helper function to set up test models with different readiness behaviors based on config parameters -setup_readiness_test_model() { - local model_name=$1 - local return_value=$2 - local delay_secs=$3 - - mkdir -p ./models/$model_name/1/ - if [ "$model_name" == "is_ready_fn_coroutine_returns_true" ]; then - cp ./test_models/readiness_coroutine_model.py ./models/$model_name/1/model.py - else - cp ./test_models/readiness_model.py ./models/$model_name/1/model.py - fi - cp ./models/identity_fp32/config.pbtxt ./models/$model_name/config.pbtxt - sed -i "s/^name:.*/name: \"$model_name\"/" ./models/$model_name/config.pbtxt - cat >> ./models/$model_name/config.pbtxt << EOF -parameters: { - key: "READINESS_FN_RETURN_VALUE" - value: { string_value: "$return_value" } -} -parameters: { - key: "READINESS_FN_DELAY_SECS" - value: { string_value: "$delay_secs" } -} -EOF -} - -# Create readiness test models using shared model.py + config parameters -setup_readiness_test_model "is_ready_fn_returns_true" "true" "0.1" -setup_readiness_test_model "is_ready_fn_returns_false" "false" "0.1" -setup_readiness_test_model "is_ready_fn_raises_error" "exception" "0.1" -setup_readiness_test_model "is_ready_fn_returns_non_boolean" "non_boolean" "0.1" -setup_readiness_test_model "is_ready_fn_timeout" "true" "8" -setup_readiness_test_model "is_ready_fn_coroutine_returns_true" "coroutine" "0.1" - -# Decoupled model has a unique execute() and its own config -mkdir -p ./models/is_ready_fn_returns_true_decoupled/1/ -cp ./test_models/is_ready_fn_returns_true_decoupled/model.py \ - ./models/is_ready_fn_returns_true_decoupled/1/model.py -cp ./test_models/is_ready_fn_returns_true_decoupled/config.pbtxt \ - ./models/is_ready_fn_returns_true_decoupled/config.pbtxt - -# Start server with all models -SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --log-verbose=1 --strict-readiness=false" -SERVER_LOG="./test_user_defined_model_readiness_function_server.log" -CLIENT_LOG="./test_user_defined_model_readiness_function_client.log" - -run_server -if [ "$SERVER_PID" == "0" ]; then - cat $SERVER_LOG - echo -e "\n***\n*** Failed to start $SERVER\n***" - exit 1 -fi - -set +e - -echo "Running TestUserDefinedModelReadinessFunction..." -python3 -m unittest test_model_readiness.TestUserDefinedModelReadinessFunction -v >> ${CLIENT_LOG} 2>&1 -TEST_EXIT_CODE=$? - -if [ $TEST_EXIT_CODE -ne 0 ]; then - echo -e "\n***\n*** TestUserDefinedModelReadinessFunction FAILED\n***" - cat ${CLIENT_LOG} - RET=1 -else - echo -e "\n***\n*** TestUserDefinedModelReadinessFunction PASSED\n***" -fi - -set -e -kill_server - - -# Final result -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** All Model Readiness Tests Passed\n***" -else - echo -e "\n***\n*** Model Readiness Tests FAILED\n***" -fi - -exit $RET - diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py deleted file mode 100644 index 5330eb1c83..0000000000 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ /dev/null @@ -1,518 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import queue -import threading -import time -import unittest -from functools import partial - -import numpy as np -import tritonclient.grpc as grpcclient -import tritonclient.http as httpclient -from tritonclient.utils import InferenceServerException - -URL_HTTP = "localhost:8000" -URL_GRPC = "localhost:8001" -DEFAULT_RESPONSE_TIMEOUT = 60 - - -class UserData: - def __init__(self): - self._response_queue = queue.Queue() - - -def callback(user_data, result, error): - if error: - user_data._response_queue.put(error) - else: - user_data._response_queue.put(result) - - -def prepare_infer_args(input_value): - """Create InferInput and InferRequestedOutput lists for decoupled inference.""" - input_data = np.array([[input_value]], dtype=np.int32) - infer_input = [grpcclient.InferInput("IN", input_data.shape, "INT32")] - infer_input[0].set_data_from_numpy(input_data) - outputs = [grpcclient.InferRequestedOutput("OUT")] - return infer_input, outputs - - -def collect_responses(user_data, expected_responses_count): - """ - Collect up to `expected_responses_count` responses from user_data. - """ - errors = [] - responses = [] - recv_count = 0 - while recv_count < expected_responses_count: - try: - result = user_data._response_queue.get(timeout=DEFAULT_RESPONSE_TIMEOUT) - except queue.Empty: - raise Exception( - f"No response received within {DEFAULT_RESPONSE_TIMEOUT} seconds." - ) - if type(result) == InferenceServerException: - errors.append(result) - break - else: - responses.append(result.as_numpy("OUT")[0]) - recv_count = recv_count + 1 - - return errors, responses - - -def call_inference_identity_model(model_name, protocol, client): - """Send an inference request and verify the output matches the input.""" - shape = (1, 8) - input_data = np.ones(shape, dtype=np.float32) - - if protocol == "http": - inputs = [httpclient.InferInput("INPUT0", input_data.shape, "FP32")] - else: - inputs = [grpcclient.InferInput("INPUT0", input_data.shape, "FP32")] - - inputs[0].set_data_from_numpy(input_data) - result = client.infer(model_name, inputs) - output_data = result.as_numpy("OUTPUT0") - - np.testing.assert_array_almost_equal( - input_data, - output_data, - err_msg=f"Inference output mismatch for {model_name}", - ) - - -class TestModelReadiness(unittest.TestCase): - def setUp(self): - self.model_name = "identity_fp32" - self.client_http = httpclient.InferenceServerClient(url=URL_HTTP) - self.client_grpc = grpcclient.InferenceServerClient(url=URL_GRPC) - - def test_model_ready(self): - # Check HTTP - try: - is_ready = self.client_http.is_model_ready(self.model_name) - self.assertTrue( - is_ready, f"[HTTP] Model {self.model_name} should be READY but is NOT" - ) - call_inference_identity_model(self.model_name, "http", self.client_http) - except Exception as e: - self.fail(f"[HTTP] Unexpected error: {str(e)}") - - # Check gRPC - try: - is_ready = self.client_grpc.is_model_ready(self.model_name) - self.assertTrue( - is_ready, f"[gRPC] Model {self.model_name} should be READY but is NOT" - ) - call_inference_identity_model(self.model_name, "grpc", self.client_grpc) - except Exception as e: - self.fail(f"[gRPC] Unexpected error: {str(e)}") - - def test_model_not_ready(self): - # Check HTTP - try: - is_ready = self.client_http.is_model_ready(self.model_name) - self.assertFalse( - is_ready, - f"[HTTP] Model {self.model_name} should be NOT READY but is READY", - ) - except Exception as e: - self.fail(f"[HTTP] Unexpected error: {str(e)}") - - # Check gRPC - try: - is_ready = self.client_grpc.is_model_ready(self.model_name) - self.assertFalse( - is_ready, - f"[gRPC] Model {self.model_name} should be NOT READY but is READY.", - ) - except Exception as e: - self.fail(f"[gRPC] Unexpected error: {str(e)}") - - -class TestUserDefinedModelReadinessFunction(unittest.TestCase): - """ - Test user-defined is_ready() function - """ - - def setUp(self): - self.client_http = httpclient.InferenceServerClient(url=URL_HTTP) - self.client_grpc = grpcclient.InferenceServerClient(url=URL_GRPC) - - def _run_inference_decoupled(self, index, model_name, expected_responses_count): - """Send a decoupled streaming inference request and verify responses.""" - user_data = UserData() - with grpcclient.InferenceServerClient(URL_GRPC) as triton_client: - try: - inputs, outputs = prepare_infer_args(expected_responses_count) - triton_client.start_stream(callback=partial(callback, user_data)) - triton_client.async_stream_infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - - # Collect and verify responses - errors, responses = collect_responses( - user_data, expected_responses_count - ) - self.assertEqual( - len(responses), - expected_responses_count, - f"Index: {index} - Expected {expected_responses_count} responses, got {len(responses)}", - ) - self.assertEqual( - len(errors), - 0, - f"Index: {index} - Expected 0 errors, got {len(errors)}", - ) - - # Verify correctness of successful responses - for idx, output in enumerate(responses): - self.assertEqual( - output, - expected_responses_count, - msg=f"Response {idx} has incorrect value - {output}", - ) - finally: - triton_client.stop_stream() - - def test_multiple_concurrent_ready_and_infer_requests_decoupled(self): - model_name = "is_ready_fn_returns_true_decoupled" - num_requests = 16 - response_count = 8 - readiness_errors = [] - infer_errors = [] - - def readiness_wrapper(index, model_name): - try: - with grpcclient.InferenceServerClient(url=URL_GRPC) as triton_client: - is_ready = triton_client.is_model_ready(model_name) - if not is_ready: - raise AssertionError( - f"Index: {index} - GRPC client - Model {model_name} should be READY" - ) - except Exception as e: - readiness_errors.append((index, str(e))) - - def inference_wrapper(index, model_name): - try: - self._run_inference_decoupled(index, model_name, response_count) - except Exception as e: - infer_errors.append((index, str(e))) - - # Launch concurrent threads - threads = [] - for i in range(num_requests): - # Start threads with slight delay - time.sleep(0.1) - t1 = threading.Thread( - target=inference_wrapper, args=(i, model_name), name=f"infer-{i}" - ) - t2 = threading.Thread( - target=readiness_wrapper, args=(i, model_name), name=f"ready-{i}" - ) - threads.extend([t1, t2]) - t1.start() - t2.start() - - # Wait for all requests to complete - for t in threads: - t.join(timeout=120) - - for t in threads: - self.assertFalse(t.is_alive(), f"Threads are not completed: {t.name}") - - self.assertEqual( - len(readiness_errors), 0, f"Readiness errors: {readiness_errors}" - ) - self.assertEqual(len(infer_errors), 0, f"Inference errors: {infer_errors}") - - def test_is_ready_coroutine_returns_true(self): - model_name = "is_ready_fn_coroutine_returns_true" - for _ in range(5): - self.assertTrue( - self.client_http.is_model_ready(model_name), - f"HTTP - Model {model_name} (coroutine) should be READY", - ) - self.assertTrue( - self.client_grpc.is_model_ready(model_name), - f"gRPC - Model {model_name} (coroutine) should be READY", - ) - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_is_ready_returns_true(self): - model_name = "is_ready_fn_returns_true" - num_requests = 10 - - # Send multiple requests in sequence to ensure consistent behavior - for i in range(num_requests): - self.assertTrue( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be READY", - ) - self.assertTrue( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be READY", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_is_ready_returns_false(self): - model_name = "is_ready_fn_returns_false" - num_requests = 10 - - # Send multiple requests in sequence to ensure consistent behavior - for i in range(num_requests): - self.assertFalse( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be NOT READY", - ) - self.assertFalse( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be NOT READY", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_is_ready_raises_exception(self): - model_name = "is_ready_fn_raises_error" - num_requests = 10 - - # Send multiple requests in sequence to ensure consistent behavior - for i in range(num_requests): - self.assertFalse( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be NOT READY (exception)", - ) - self.assertFalse( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (exception)", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - # Verify a healthy model is still ready to confirm server stability. - model_name = "is_ready_fn_returns_true" - for i in range(num_requests): - self.assertTrue( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be READY", - ) - self.assertTrue( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be READY", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_is_ready_returns_non_boolean(self): - model_name = "is_ready_fn_returns_non_boolean" - num_requests = 10 - - # Send multiple requests in sequence to ensure consistent behavior - for i in range(num_requests): - self.assertFalse( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be NOT READY (wrong return type)", - ) - self.assertFalse( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (wrong return type)", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - # Verify a healthy model is still ready to confirm server stability. - model_name = "is_ready_fn_returns_true" - for i in range(num_requests): - self.assertTrue( - self.client_http.is_model_ready(model_name), - f"iteration {i} - HTTP client - Model {model_name} should be READY", - ) - self.assertTrue( - self.client_grpc.is_model_ready(model_name), - f"iteration {i} - GRPC client - Model {model_name} should be READY", - ) - - # Verify inference is unaffected by readiness checks. - call_inference_identity_model(model_name, "http", self.client_http) - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_is_ready_takes_long_time(self): - model_name = "is_ready_fn_timeout" - num_requests = 10 - - # Send multiple requests in sequence to ensure consistent behavior - for i in range(num_requests): - # This call should time out and return NOT_READY. - # Note: the stub will continue running is_ready() - # in the background (similar to the inference flow) - # even after the backend readiness timeout expires. - is_ready = self.client_http.is_model_ready(model_name) - self.assertFalse( - is_ready, - f"iteration {i} - HTTP client - Model {model_name} should timeout and return NOT READY", - ) - - call_inference_identity_model(model_name, "http", self.client_http) - - # This call should not create another internal IPC message. - # It must wait for the in-flight readiness check - # and return READY once that check completes. - is_ready = self.client_grpc.is_model_ready(model_name) - self.assertTrue( - is_ready, - f"iteration {i} - GRPC client - Model {model_name} should be READY", - ) - - call_inference_identity_model(model_name, "grpc", self.client_grpc) - - def test_multiple_concurrent_ready_and_infer_requests(self): - model_name = "is_ready_fn_returns_true" - ready_results = {"http": [], "grpc": []} - ready_errors = {"http": [], "grpc": []} - infer_results = {"http": [], "grpc": []} - infer_errors = {"http": [], "grpc": []} - num_requests = 16 - - def check_model_readiness(protocol, index): - try: - if protocol == "http": - with httpclient.InferenceServerClient(url=URL_HTTP) as client_http: - is_ready = client_http.is_model_ready(model_name) - ready_results["http"].append((index, is_ready)) - else: - with grpcclient.InferenceServerClient(url=URL_GRPC) as client_grpc: - is_ready = client_grpc.is_model_ready(model_name) - ready_results["grpc"].append((index, is_ready)) - except Exception as e: - ready_errors[protocol].append((index, str(e))) - - def do_inference(protocol, index): - try: - if protocol == "http": - with httpclient.InferenceServerClient(url=URL_HTTP) as client_http: - start = time.time() - call_inference_identity_model(model_name, protocol, client_http) - elapsed = time.time() - start - infer_results["http"].append((index, True, elapsed)) - else: - with grpcclient.InferenceServerClient(url=URL_GRPC) as client_grpc: - start = time.time() - call_inference_identity_model(model_name, protocol, client_grpc) - elapsed = time.time() - start - infer_results["grpc"].append((index, True, elapsed)) - except Exception as e: - infer_errors[protocol].append((index, str(e))) - - # Launch concurrent readiness and inference requests. - http_threads = [] - for i in range(num_requests): - t1 = threading.Thread(target=check_model_readiness, args=("http", i)) - t2 = threading.Thread(target=do_inference, args=("http", i)) - http_threads.extend([t1, t2]) - t1.start() - t2.start() - - # Wait for all requests to complete - for t in http_threads: - t.join(timeout=60) - - for t in http_threads: - self.assertFalse(t.is_alive(), f"HTTP threads are not completed") - - grpc_threads = [] - for i in range(num_requests): - t1 = threading.Thread(target=check_model_readiness, args=("grpc", i)) - t2 = threading.Thread(target=do_inference, args=("grpc", i)) - grpc_threads.extend([t1, t2]) - t1.start() - t2.start() - - # Wait for all requests to complete - for t in grpc_threads: - t.join(timeout=60) - - for t in grpc_threads: - self.assertFalse(t.is_alive(), f"gRPC threads are not completed") - - # Verify no errors in readiness checks - self.assertEqual( - len(ready_errors["http"]), 0, f"HTTP errors: {ready_errors['http']}" - ) - self.assertEqual( - len(ready_errors["grpc"]), 0, f"gRPC errors: {ready_errors['grpc']}" - ) - self.assertEqual( - len(ready_results["http"]), - num_requests, - f"Expected {num_requests} HTTP results", - ) - self.assertEqual( - len(ready_results["grpc"]), - num_requests, - f"Expected {num_requests} gRPC results", - ) - - # All should be True - for idx, ready in ready_results["http"]: - self.assertTrue(ready, f"HTTP check {idx} should be ready") - for idx, ready in ready_results["grpc"]: - self.assertTrue(ready, f"gRPC check {idx} should be ready") - - # Verify no errors in inference - self.assertEqual( - len(infer_errors["http"]), 0, f"Errors occurred: {infer_errors['http']}" - ) - self.assertEqual( - len(infer_errors["grpc"]), 0, f"Errors occurred: {infer_errors['grpc']}" - ) - self.assertEqual( - len(infer_results["http"]), - num_requests, - f"Expected {num_requests} HTTP inference results", - ) - self.assertEqual( - len(infer_results["grpc"]), - num_requests, - f"Expected {num_requests} gRPC inference results", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py b/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py deleted file mode 100644 index e2a4dfcb0d..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import time - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Decoupled model that produces N responses based on input value. - """ - - def execute(self, requests): - for request in requests: - # Get input - number of responses to produce - in_tensor = pb_utils.get_input_tensor_by_name(request, "IN") - count = in_tensor.as_numpy().item() - - response_sender = request.get_response_sender() - out_tensor = pb_utils.Tensor("OUT", np.array([count], dtype=np.int32)) - - # Produce 'count' responses, each with 'count' as the output value - for i in range(count): - # Simulate some processing delay - time.sleep(0.1) - response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) - response_sender.send(response) - - # Send final flag - response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) - - return None - - def is_ready(self) -> bool: - # Simulate some processing delay - time.sleep(0.2) - return True diff --git a/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py deleted file mode 100644 index ce33afe700..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import asyncio -import json - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Parameterized test model for async is_ready() testing. - - Behavior is controlled via config.pbtxt parameters: - READINESS_FN_DELAY_SECS - seconds to await before returning (e.g. "0.1") - """ - - def initialize(self, args): - model_config = json.loads(args["model_config"]) - params = model_config.get("parameters", {}) - self.readiness_delay_secs = float( - params.get("READINESS_FN_DELAY_SECS", {}).get("string_value", "0.1") - ) - - def execute(self, requests): - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - async def is_ready(self): - await asyncio.sleep(self.readiness_delay_secs) - return True diff --git a/qa/L0_backend_python/model_readiness/test_models/readiness_model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_model.py deleted file mode 100644 index 444bd9e983..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/readiness_model.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import time - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Parameterized test model for user-defined is_ready() testing. - - Behavior is controlled via config.pbtxt parameters: - READINESS_FN_RETURN_VALUE - "true", "false", "exception", or "non_boolean" - READINESS_FN_DELAY_SECS - seconds to sleep before returning (e.g. "0.1") - """ - - def initialize(self, args): - model_config = json.loads(args["model_config"]) - params = model_config.get("parameters", {}) - self.readiness_return_value = params.get("READINESS_FN_RETURN_VALUE", {}).get( - "string_value", "true" - ) - self.readiness_delay_secs = float( - params.get("READINESS_FN_DELAY_SECS", {}).get("string_value", "0") - ) - - def execute(self, requests): - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - def is_ready(self): - if self.readiness_delay_secs > 0: - time.sleep(self.readiness_delay_secs) - - if self.readiness_return_value == "true": - return True - elif self.readiness_return_value == "false": - return False - elif self.readiness_return_value == "exception": - raise RuntimeError("Internal check failed – model is not ready") - elif self.readiness_return_value == "non_boolean": - return "ready" - return True diff --git a/qa/L0_backend_python/python_test.py b/qa/L0_backend_python/python_test.py index 135aba3060..c2512600e2 100755 --- a/qa/L0_backend_python/python_test.py +++ b/qa/L0_backend_python/python_test.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,7 +33,6 @@ import os import unittest -import ml_dtypes import numpy as np import requests as httpreq import shm_util @@ -373,7 +372,9 @@ def test_bf16(self): with httpclient.InferenceServerClient( f"{_tritonserver_ipaddr}:8000" ) as client: - np_input = np.ones(shape, dtype=ml_dtypes.bfloat16) + # NOTE: Client will truncate FP32 to BF16 internally + # since numpy has no built-in BF16 representation. + np_input = np.ones(shape, dtype=np.float32) inputs = [ httpclient.InferInput( "INPUT0", np_input.shape, "BF16" @@ -389,8 +390,10 @@ def test_bf16(self): np_output = result.as_numpy("OUTPUT0") self.assertIsNotNone(np_output) - self.assertEqual(np_output.dtype, np_input.dtype) - self.assertTrue(np.array_equal(np_output, np_input)) + # BF16 tensors are held in FP32 when converted to numpy due to + # lack of native BF16 support in numpy, so verify that. + self.assertEqual(np_output.dtype, np.float32) + self.assertTrue(np.allclose(np_output, np_input)) def test_infer_pytorch(self): # FIXME: This model requires torch. Because windows tests are not run in a docker diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh index ce6be322ba..c3d0a0aed7 100755 --- a/qa/L0_backend_python/setup_python_enviroment.sh +++ b/qa/L0_backend_python/setup_python_enviroment.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -56,10 +56,10 @@ conda update -n base -c defaults conda -y # been setup correctly. if [ ${PYTHON_ENV_VERSION} = "11" ]; then create_conda_env "3.11" "python-3-11" - conda install pytorch=2.8.0 -y + conda install torch=2.6.0 -y conda install -c conda-forge libstdcxx-ng=14 -y conda install numpy=1.23.5 -y - EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and PyTorch version is 2.8.0" + EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and PyTorch version is 2.6.0" create_python_backend_stub conda-pack -o python3.11.tar.gz path_to_conda_pack="$PWD/python-3-11" @@ -99,7 +99,6 @@ echo "python environment 3.${PYTHON_ENV_VERSION}" # copy the stub out to /opt/tritonserver/backends/python/triton_python_backend_stub cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/triton_python_backend_stub # Set up environment and stub for each test -apt-get update -qq && apt-get install -y software-properties-common add-apt-repository ppa:deadsnakes/ppa -y apt-get update && apt-get -y install \ "python3.${PYTHON_ENV_VERSION}-dev" \ @@ -109,12 +108,11 @@ rm -f /usr/bin/python3 && \ ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3 pip3 install --upgrade requests numpy virtualenv protobuf find /opt/tritonserver/qa/pkgs/ -maxdepth 1 -type f -name \ - "tritonclient-*-py3-none-any.whl" | xargs printf -- '%s[all]' | \ + "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ xargs pip3 install --upgrade # Build triton-shm-monitor for the test cd python_backend && rm -rf install build && mkdir build && cd build && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=$PWD/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index bc739a6219..b2e6b8f034 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -65,6 +65,9 @@ export PYTHON_ENV_VERSION=${PYTHON_ENV_VERSION:="12"} export PYTHON_BACKEND_REPO_TAG=$PYTHON_BACKEND_REPO_TAG BASE_SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=1" +# Set the default byte size to 5MBs to avoid going out of shared memory. The +# environment that this job runs on has only 1GB of shared-memory available. +SERVER_ARGS="$BASE_SERVER_ARGS --backend-config=python,shm-default-byte-size=5242880" CLIENT_PY=./python_test.py CLIENT_LOG="./client.log" @@ -174,9 +177,6 @@ fi pip3 install pytest requests virtualenv -# Set the default byte size to 5MBs to avoid going out of shared memory. The -# environment that this job runs on has only 1GB of shared-memory available. -SERVER_ARGS="$BASE_SERVER_ARGS --allow-client-shm=true --backend-config=python,shm-default-byte-size=5242880" prev_num_pages=`get_shm_pages` run_server if [ "$SERVER_PID" == "0" ]; then @@ -205,7 +205,6 @@ and shared memory pages after starting triton equals to $current_num_pages \n*** RET=1 fi -SERVER_ARGS="$BASE_SERVER_ARGS --backend-config=python,shm-default-byte-size=5242880" prev_num_pages=`get_shm_pages` # Triton non-graceful exit run_server @@ -408,65 +407,6 @@ and shared memory pages after starting triton equals to $current_num_pages \n*** exit 1 fi - -# Test model with non-existent model file -# The model.py file is intentionally not created to trigger the failure. -rm -fr ./models -mkdir -p models/non_existent_model/1 -cp ../python_models/identity_fp32/config.pbtxt ./models/non_existent_model/config.pbtxt -(cd models/non_existent_model && \ - sed -i "s/^name:.*/name: \"non_existent_model\"/" config.pbtxt) - -ERROR_MESSAGE_1="Failed to preinitialize Python stub: Python model file not found in '`pwd`/models/non_existent_model/1/model.py'" -ERROR_MESSAGE_2="failed to load 'non_existent_model'" - -for test_mode in "default" "auto_config_disabled"; do - if [ "$test_mode" == "default" ]; then - SERVER_LOG_SUFFIX="" - SERVER_ARGS=$BASE_SERVER_ARGS - else - SERVER_LOG_SUFFIX="_auto_config_disabled" - SERVER_ARGS="$BASE_SERVER_ARGS --disable-auto-complete-config" - fi - - SERVER_LOG="./non_existent_model_server${SERVER_LOG_SUFFIX}.log" - CLIENT_LOG="./non_existent_model_client${SERVER_LOG_SUFFIX}.log" - - prev_num_pages=`get_shm_pages` - run_server - if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - RET=1 - kill_server - else - if grep -q "$ERROR_MESSAGE_1" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_1\"" >> $CLIENT_LOG - else - echo -e "Not found \"$ERROR_MESSAGE_1\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG - RET=1 - fi - - if grep -q "$ERROR_MESSAGE_2" $SERVER_LOG; then - echo -e "Found \"$ERROR_MESSAGE_2\"" >> $CLIENT_LOG - else - echo -e "Not found \"$ERROR_MESSAGE_2\" in $SERVER_LOG" >> $CLIENT_LOG - cat $SERVER_LOG >> $CLIENT_LOG - RET=1 - fi - fi - - current_num_pages=`get_shm_pages` - if [ $current_num_pages -ne $prev_num_pages ]; then - cat $SERVER_LOG - ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages were not cleaned properly. -Shared memory pages before starting triton equals to $prev_num_pages -and shared memory pages after starting triton equals to $current_num_pages \n***" - exit 1 - fi -done - # Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported # Disable variants test for Jetson since already built without GPU Tensor support @@ -523,7 +463,7 @@ SUBTESTS="lifecycle argument_validation logging custom_metrics parameters" # [DLIS-6123] Disable examples test for Windows since it requires updates to the example clients if [[ ${TEST_WINDOWS} == 0 ]]; then # TODO: Reimplement restart on decoupled data pipeline and enable restart. - SUBTESTS+=" model_control examples request_rescheduling model_readiness" + SUBTESTS+=" model_control examples request_rescheduling" fi for TEST in ${SUBTESTS}; do # Run each subtest in a separate virtual environment to avoid conflicts diff --git a/qa/L0_backend_python/variants/test.sh b/qa/L0_backend_python/variants/test.sh index 3e3677cbe4..86cc793a94 100755 --- a/qa/L0_backend_python/variants/test.sh +++ b/qa/L0_backend_python/variants/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,7 +34,6 @@ rm -rf python_backend git clone ${TRITON_REPO_ORGANIZATION}/python_backend -b $PYTHON_BACKEND_REPO_TAG (cd python_backend/ && mkdir builddir && cd builddir && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DTRITON_ENABLE_GPU=OFF -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} -DTRITON_BACKEND_REPO_TAG=$TRITON_BACKEND_REPO_TAG -DTRITON_COMMON_REPO_TAG=$TRITON_COMMON_REPO_TAG -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG ../ && \ make -j18 install) diff --git a/qa/L0_backend_tutorial/test.sh b/qa/L0_backend_tutorial/test.sh index 60c98c18f8..3e59503b8c 100755 --- a/qa/L0_backend_tutorial/test.sh +++ b/qa/L0_backend_tutorial/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,7 +48,7 @@ apt update -q=2 \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ - && apt-get install -y --no-install-recommends cmake=4.0.3* cmake-data=4.0.3* \ + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* \ rapidjson-dev cmake --version @@ -62,7 +62,6 @@ git clone --single-branch --depth=1 -b $TRITON_BACKEND_REPO_TAG \ (cd backend/examples/backends/minimal && mkdir build && cd build && - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ @@ -141,7 +140,6 @@ rm -fr /opt/tritonserver/backends/minimal (cd backend/examples/backends/recommended && mkdir build && cd build && - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ diff --git a/qa/L0_batch_custom/test.sh b/qa/L0_batch_custom/test.sh index 5ed0f3a91b..13d54c25d8 100755 --- a/qa/L0_batch_custom/test.sh +++ b/qa/L0_batch_custom/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,8 +54,6 @@ TEST_RESULT_FILE='test_results.txt' TRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION:="http://github.com/triton-inference-server"} TRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG:="main"} TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG:="main"} -TRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG:="main"} - source ../common/util.sh RET=0 @@ -68,7 +66,7 @@ apt update -q=2 \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ - && apt-get install -y --no-install-recommends cmake=4.0.3* cmake-data=4.0.3* rapidjson-dev + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* rapidjson-dev cmake --version # Set up repository @@ -88,23 +86,17 @@ git clone --single-branch --depth=1 -b $TRITON_BACKEND_REPO_TAG \ (cd backend/examples/batching_strategies/volume_batching && mkdir build && cd build && - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} .. && + -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG .. && make -j4 install) (cd backend/examples/batching_strategies/single_batching && mkdir build && cd build && - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} .. && + -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG .. && make -j4 install) cp -r backend/examples/batching_strategies/volume_batching/build/libtriton_volumebatching.so models @@ -172,10 +164,8 @@ sed -i "s/${OLD_STRING}/${NEW_STRING}/g" ${FILE_PATH} (cd backend/examples/batching_strategies/volume_batching && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=`pwd`/install \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ - -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} .. && + -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_CORE_REPO_TAG=$TRITON_CORE_REPO_TAG .. && make -j4 install) cp -r backend/examples/batching_strategies/volume_batching/build/libtriton_volumebatching.so models/${MODEL_NAME}/libtriton_volumebatching.so diff --git a/qa/L0_batcher/batcher_test.py b/qa/L0_batcher/batcher_test.py index 6a133238ef..780eb1e68f 100755 --- a/qa/L0_batcher/batcher_test.py +++ b/qa/L0_batcher/batcher_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -74,7 +74,6 @@ _ragged_batch_supported_trials.append("libtorch") _max_queue_delay_ms = 10000 -_max_expected_response_ms = 3000 _deferred_exceptions_lock = threading.Lock() _deferred_exceptions = [] @@ -312,13 +311,13 @@ def test_static_batch_preferred(self): self.check_response( trial, 2, - (_max_expected_response_ms, None), + (3000, None), precreated_shm_regions=precreated_shm_regions, ) self.check_response( trial, 6, - (_max_expected_response_ms, None), + (3000, None), precreated_shm_regions=precreated_shm_regions, ) self.check_deferred_exception() @@ -393,7 +392,7 @@ def test_static_batch_gt_max_preferred(self): self.check_response( trial, 7, - (_max_expected_response_ms, None), + (3000, None), precreated_shm_regions=precreated_shm_regions, ) self.check_deferred_exception() @@ -476,7 +475,7 @@ def test_multi_batch_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "input_size": 16, "shm_region_names": shm0_region_names, @@ -601,7 +600,7 @@ def test_multi_batch_not_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -611,7 +610,7 @@ def test_multi_batch_not_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms * 2, None)), + args=(trial, 3, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -679,7 +678,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -689,7 +688,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms * 2, None)), + args=(trial, 3, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -699,7 +698,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "input_size": 8, "shm_region_names": shm2_region_names, @@ -710,7 +709,7 @@ def test_multi_batch_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 5, (_max_expected_response_ms * 2, None)), + args=(trial, 5, (6000, None)), kwargs={ "input_size": 8, "shm_region_names": shm3_region_names, @@ -757,7 +756,7 @@ def test_multi_batch_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms, None)), + args=(trial, 3, (3000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -767,7 +766,7 @@ def test_multi_batch_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 7, (_max_expected_response_ms, None)), + args=(trial, 7, (3000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -814,7 +813,7 @@ def test_multi_batch_sum_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms, None)), + args=(trial, 3, (3000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -871,7 +870,7 @@ def test_multi_same_output0(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm0_region_names, @@ -882,7 +881,7 @@ def test_multi_same_output0(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm1_region_names, @@ -925,7 +924,7 @@ def test_multi_same_output1(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm0_region_names, @@ -936,7 +935,7 @@ def test_multi_same_output1(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm1_region_names, @@ -980,7 +979,7 @@ def test_multi_different_outputs(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "requested_outputs": ("OUTPUT0",), "shm_region_names": shm0_region_names, @@ -991,7 +990,7 @@ def test_multi_different_outputs(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "requested_outputs": ("OUTPUT1",), "shm_region_names": shm1_region_names, @@ -1032,7 +1031,7 @@ def test_multi_different_output_order(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "requested_outputs": ("OUTPUT0", "OUTPUT1"), "shm_region_names": shm0_region_names, @@ -1042,7 +1041,7 @@ def test_multi_different_output_order(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "requested_outputs": ("OUTPUT1", "OUTPUT0"), "shm_region_names": shm1_region_names, @@ -1091,7 +1090,7 @@ def test_multi_batch_delayed_sum_gt_max_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms * 2, None)), + args=(trial, 3, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1188,7 +1187,7 @@ def test_multi_batch_delayed_use_max_batch(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1246,7 +1245,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1256,7 +1255,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 3, (_max_expected_response_ms, None)), + args=(trial, 3, (3000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1266,7 +1265,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms, None)), + args=(trial, 1, (3000, None)), kwargs={ "input_size": 8, "shm_region_names": shm2_region_names, @@ -1277,7 +1276,7 @@ def test_multi_batch_delayed_preferred_different_shape(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 5, (_max_expected_response_ms, None)), + args=(trial, 5, (3000, None)), kwargs={ "input_size": 8, "shm_region_names": shm3_region_names, @@ -1339,7 +1338,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1349,7 +1348,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1359,7 +1358,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1369,7 +1368,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1379,7 +1378,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1389,7 +1388,7 @@ def test_multi_batch_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm5_region_names, "precreated_shm_regions": precreated_shm5_regions, @@ -1440,7 +1439,7 @@ def test_multi_batch_use_best_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1450,7 +1449,7 @@ def test_multi_batch_use_best_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1561,7 +1560,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1571,7 +1570,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1581,7 +1580,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1591,7 +1590,7 @@ def test_preferred_batch_only_aligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1646,7 +1645,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1656,7 +1655,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1666,7 +1665,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1676,7 +1675,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1686,7 +1685,7 @@ def test_preferred_batch_only_unaligned(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1747,7 +1746,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1757,7 +1756,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1767,7 +1766,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, @@ -1777,7 +1776,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm3_region_names, "precreated_shm_regions": precreated_shm3_regions, @@ -1787,7 +1786,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm4_region_names, "precreated_shm_regions": precreated_shm4_regions, @@ -1797,7 +1796,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm5_region_names, "precreated_shm_regions": precreated_shm5_regions, @@ -1807,7 +1806,7 @@ def test_preferred_batch_only_use_biggest_preferred(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm6_region_names, "precreated_shm_regions": precreated_shm6_regions, @@ -1855,7 +1854,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm0_region_names, "precreated_shm_regions": precreated_shm0_regions, @@ -1865,7 +1864,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm1_region_names, "precreated_shm_regions": precreated_shm1_regions, @@ -1875,7 +1874,7 @@ def test_preferred_batch_only_use_no_preferred_size(self): threads.append( threading.Thread( target=self.check_response, - args=(trial, 1, (_max_expected_response_ms * 2, None)), + args=(trial, 1, (6000, None)), kwargs={ "shm_region_names": shm2_region_names, "precreated_shm_regions": precreated_shm2_regions, diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh index a17594f8f9..136fbe586d 100755 --- a/qa/L0_batcher/test.sh +++ b/qa/L0_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,6 +42,7 @@ fi # can fail when the requests are distributed to multiple devices. export CUDA_VISIBLE_DEVICES=0 +CLIENT_LOG="./client.log" BATCHER_TEST=batcher_test.py VERIFY_TIMESTAMPS=verify_timestamps.py TEST_RESULT_FILE='test_results.txt' @@ -103,9 +104,6 @@ else fi SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR}" -if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then - SERVER_ARGS_EXTRA="${SERVER_ARGS_EXTRA} --allow-client-shm=true" -fi source ../common/util.sh RET=0 @@ -267,77 +265,17 @@ if [[ $BACKENDS == *"libtorch"* ]]; then dynamic_batching { preferred_batch_size: [ 2, 6 ], max_queue_delay_microseconds: 10000000 }" >> config.pbtxt) fi -warmup_cuda_cache() { - local backend=$1 - local batch_size=$2 - local model="${backend}_float32_float32_float32" - local n=$((16 * batch_size)) - local input0_data=$(printf '1,%.0s' $(seq 1 $n) | sed 's/,$//') - local input1_data=$(printf '2,%.0s' $(seq 1 $n) | sed 's/,$//') - - SERVER_ARGS="--model-repository=$MODELDIR/models --model-control-mode=explicit --load-model=$model" - SERVER_LOG="./warmup_cuda_cache.server.log" - - run_server - - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - - curl -X POST "http://localhost:8000/v2/models/${model}/versions/1/infer" \ - -H "Content-Type: application/json" \ - -d '{ - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": ['"$batch_size"', 16], - "data": ['"$input0_data"'] - }, - { - "name": "INPUT1", - "datatype": "FP32", - "shape": ['"$batch_size"', 16], - "data": ['"$input1_data"'] - } - ], - "outputs": [ - { "name": "OUTPUT0" }, - { "name": "OUTPUT1" } - ] - }' - - EXIT_CODE=$? - if [ $EXIT_CODE -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - echo -e "\n***\n*** exit code: $EXIT_CODE\n***" - RET=1 - fi - set -e - - kill_server -} - -# [TRI-830] Send a simple request to warmup CUDA_CACHE for GB300 before testing. -if nvidia-smi --query-gpu=name --format=csv,noheader | grep -qiF 'GB300'; then - warmup_cuda_cache onnx 2 -fi - # Need to launch the server for each test so that the model status is # reset (which is used to make sure the correctly batch size was used # for execution). Test everything with fixed-tensor-size models and # variable-tensor-size models. + for model_type in FIXED VARIABLE; do export BATCHER_TYPE=$model_type MODEL_PATH=models && [[ "$model_type" == "VARIABLE" ]] && MODEL_PATH=var_models for i in $NO_DELAY_TESTS ; do SERVER_ARGS="--model-repository=$MODELDIR/$MODEL_PATH ${SERVER_ARGS_EXTRA}" SERVER_LOG="./$i.$model_type.server.log" - CLIENT_LOG="./$i.$model_type.client.log" if [ "$TEST_VALGRIND" -eq 1 ]; then LEAKCHECK_LOG="./$i.$model_type.valgrind.log" diff --git a/qa/L0_buffer_attributes/test.sh b/qa/L0_buffer_attributes/test.sh index d07c63e11e..7e2f35d837 100755 --- a/qa/L0_buffer_attributes/test.sh +++ b/qa/L0_buffer_attributes/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -52,7 +52,7 @@ export CUDA_VISIBLE_DEVICES=0 rm -fr *.log SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models --allow-client-shm=true" +SERVER_ARGS="--model-repository=`pwd`/models" SERVER_LOG="./inference_server.log" run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/L0_client_build_variants/test.sh b/qa/L0_client_build_variants/test.sh index cafc55acc7..a64d698526 100755 --- a/qa/L0_client_build_variants/test.sh +++ b/qa/L0_client_build_variants/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -38,7 +38,7 @@ apt update -q=2 \ && . /etc/os-release \ && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ && apt-get update -q=2 \ - && apt-get install -y --no-install-recommends cmake=4.0.3* cmake-data=4.0.3* + && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* cmake --version @@ -50,15 +50,8 @@ mkdir -p /workspace/build # Build without GPU support # TRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION:="http://github.com/triton-inference-server"} -TRITON_BACKEND_REPO_TAG=${TRITON_BACKEND_REPO_TAG:="main"} -TRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG:="main"} -TRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG:="main"} - -export CMAKE_POLICY_VERSION_MINIMUM=3.5 - (cd /workspace/build && \ rm -fr cc-clients java-clients python-clients && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_ENABLE_CC_HTTP=ON \ -DTRITON_ENABLE_CC_GRPC=ON \ @@ -88,7 +81,6 @@ fi # (cd /workspace/build && \ rm -fr cc-clients python-clients && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_ENABLE_CC_HTTP=OFF \ -DTRITON_ENABLE_CC_GRPC=ON \ @@ -116,7 +108,6 @@ fi # (cd /workspace/build && \ rm -fr cc-clients python-clients && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ -DTRITON_ENABLE_CC_HTTP=ON \ -DTRITON_ENABLE_CC_GRPC=OFF \ @@ -138,6 +129,84 @@ else exit 1 fi +# TODO: TPRD-342 These tests should be PA CI test +# cases not Triton test cases +rm -fr /workspace/build +mkdir -p /workspace/build +# +# Build without C API in Perf Analyzer +# +(cd /workspace/build && \ + cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ + -DTRITON_ENABLE_CC_HTTP=ON \ + -DTRITON_ENABLE_CC_GRPC=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_C_API=OFF \ + -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ + -DTRITON_ENABLE_GPU=ON \ + -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ + -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ + -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ + /workspace/perf_analyzer && \ + make -j16 perf-analyzer) +if [ $? -eq 0 ]; then + echo -e "\n***\n*** No-CAPI Passed\n***" +else + echo -e "\n***\n*** No-CAPI FAILED\n***" + exit 1 +fi + +# +# Build without TensorFlow Serving in Perf Analyzer +# +(cd /workspace/build && \ + rm -fr cc_clients perf_analyzer && \ + cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ + -DTRITON_ENABLE_CC_HTTP=ON \ + -DTRITON_ENABLE_CC_GRPC=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TFS=OFF \ + -DTRITON_ENABLE_PERF_ANALYZER_TS=ON \ + -DTRITON_ENABLE_GPU=ON \ + -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ + -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ + -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ + /workspace/perf_analyzer && \ + make -j16 perf-analyzer) +if [ $? -eq 0 ]; then + echo -e "\n***\n*** No-TF-Serving Passed\n***" +else + echo -e "\n***\n*** No-TF-Serving FAILED\n***" + exit 1 +fi + +# +# Build without TorchServe in Perf Analyzer +# +(cd /workspace/build && \ + rm -fr cc_clients perf_analyzer && \ + cmake -DCMAKE_INSTALL_PREFIX=/workspace/install \ + -DTRITON_ENABLE_CC_HTTP=ON \ + -DTRITON_ENABLE_CC_GRPC=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_C_API=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TFS=ON \ + -DTRITON_ENABLE_PERF_ANALYZER_TS=OFF \ + -DTRITON_ENABLE_GPU=ON \ + -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ + -DTRITON_COMMON_REPO_TAG=${TRITON_COMMON_REPO_TAG} \ + -DTRITON_CORE_REPO_TAG=${TRITON_CORE_REPO_TAG} \ + -DTRITON_CLIENT_REPO_TAG=${TRITON_CLIENT_REPO_TAG} \ + /workspace/perf_analyzer && \ + make -j16 perf-analyzer) +if [ $? -eq 0 ]; then + echo -e "\n***\n*** No-TorchServe Passed\n***" +else + echo -e "\n***\n*** No-TorchServe FAILED\n***" + exit 1 +fi + set -e echo -e "\n***\n*** Test Passed\n***" diff --git a/qa/L0_client_nobatch/test.sh b/qa/L0_client_nobatch/test.sh index 0c01bafb70..cb1a05d660 100755 --- a/qa/L0_client_nobatch/test.sh +++ b/qa/L0_client_nobatch/test.sh @@ -47,7 +47,7 @@ EXPECTED_NUM_TESTS="4" DATADIR=/data/inferenceserver/${REPO_VERSION} MODELDIR="${PWD}/qa_model_repository" -rm -rf ${MODELDIR} && mkdir -p ${MODELDIR} && cp -r ${DATADIR}/qa_model_repository/onnx_* ${MODELDIR}/. # Note there is a coupling in ./client_test.py +rm -rf ${MODELDIR} && cp -r "${DATADIR}/qa_model_repository/onnx_*" ${MODELDIR} # Note there is a coupling in ./client_test.py SERVER=/opt/tritonserver/bin/tritonserver SERVER_ARGS="--model-repository=${MODELDIR}" SERVER_LOG="./inference_server.log" diff --git a/qa/L0_client_timeout/test.sh b/qa/L0_client_timeout/test.sh index fe11549730..f250dc9fa3 100755 --- a/qa/L0_client_timeout/test.sh +++ b/qa/L0_client_timeout/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -241,7 +241,7 @@ wait $SERVER_PID # Test all APIs other than infer export TRITONSERVER_SERVER_DELAY_GRPC_RESPONSE_SEC=2 -SERVER_ARGS="${SERVER_ARGS} --model-control-mode=explicit --allow-client-shm=true --load-model=custom_identity_int32 --log-verbose 2" +SERVER_ARGS="${SERVER_ARGS} --model-control-mode=explicit --load-model=custom_identity_int32 --log-verbose 2" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" diff --git a/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py b/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py index 12ade110f0..cce6f4f63e 100755 --- a/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py +++ b/qa/L0_cuda_shared_memory/cuda_shared_memory_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -153,39 +153,6 @@ def _cleanup_shm_handles(self): class CudaSharedMemoryTest(CudaSharedMemoryTestBase): - def test_client_shm_disabled_by_default(self): - # When the server is started without --allow-client-shm, registration and - # unregistration are rejected but querying status remains allowed (empty). - shm_op0_handle = cshm.create_shared_memory_region("dummy_data", 8, 0) - self._shm_handles.append(shm_op0_handle) - - shm_status_before = self.triton_client.get_cuda_shared_memory_status() - if self.protocol == "http": - self.assertEqual(len(shm_status_before), 0) - else: - self.assertEqual(len(shm_status_before.regions), 0) - - with self.assertRaisesRegex( - InferenceServerException, - "Client shared memory is disabled", - ): - self.triton_client.register_cuda_shared_memory( - "dummy_data", cshm.get_raw_handle(shm_op0_handle), 0, 8 - ) - - with self.assertRaisesRegex( - InferenceServerException, - "Client shared memory is disabled", - ): - self.triton_client.unregister_cuda_shared_memory("dummy_data") - - shm_status_after = self.triton_client.get_cuda_shared_memory_status() - self.assertEqual( - shm_status_before, - shm_status_after, - "CUDA shared memory status must be unchanged after failed register/unregister", - ) - def test_invalid_create_shm(self): # Raises error since tried to create invalid cuda shared memory region with self.assertRaisesRegex( @@ -756,11 +723,7 @@ def test_exceeds_cshm_handle_size_limit(self): try: error_message = response.json().get("error", "") self.assertIn( - "request JSON size", - error_message, - ) - self.assertIn( - "exceeds the maximum allowed input size", + "'raw_handle' exceeds the maximum allowed data size limit INT_MAX", error_message, ) except ValueError: diff --git a/qa/L0_cuda_shared_memory/test.sh b/qa/L0_cuda_shared_memory/test.sh index 1802731cd3..962aa106cb 100755 --- a/qa/L0_cuda_shared_memory/test.sh +++ b/qa/L0_cuda_shared_memory/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -37,45 +37,6 @@ source ../common/util.sh RET=0 rm -fr *.log -# Test that shared memory registration/unregistration is rejected and status -# query is allowed when --allow-client-shm is not set (default is false). -for client_type in http grpc; do - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" - SERVER_LOG="./test_client_shm_disabled_by_default.$client_type.server.log" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - export CLIENT_TYPE=$client_type - TEST_CLIENT_LOG="./test_client_shm_disabled_by_default.$client_type.client.log" - echo "Test: test_client_shm_disabled_by_default, client type: $client_type" >>$TEST_CLIENT_LOG - - set +e - python $SHM_TEST CudaSharedMemoryTest.test_client_shm_disabled_by_default >>$TEST_CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - cat $TEST_CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $TEST_RESULT_FILE - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi - fi - set -e - - kill $SERVER_PID - wait $SERVER_PID -done - -# Test CUDA shared memory registration with --allow-client-shm=true -SERVER_ARGS_EXTRA="--allow-client-shm=true" - for i in \ test_invalid_create_shm \ test_valid_create_set_register \ @@ -91,7 +52,7 @@ for i in \ test_infer_offset_out_of_bound \ test_infer_byte_size_out_of_bound; do for client_type in http grpc; do - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 ${SERVER_ARGS_EXTRA}" + SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" SERVER_LOG="./$i.$client_type.server.log" run_server if [ "$SERVER_PID" == "0" ]; then @@ -127,7 +88,7 @@ for i in \ test_exceeds_cshm_handle_size_limit \ test_invalid_small_cshm_handle \ test_valid_cshm_handle; do - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 ${SERVER_ARGS_EXTRA}" + SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" SERVER_LOG="./$i.server.log" CLIENT_LOG="./$i.client.log" run_server diff --git a/qa/L0_decoupled/decoupled_test.py b/qa/L0_decoupled/decoupled_test.py index edb5d8c48a..d7bc59f5c7 100755 --- a/qa/L0_decoupled/decoupled_test.py +++ b/qa/L0_decoupled/decoupled_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,7 +32,6 @@ import os import queue -import threading import time import unittest from functools import partial @@ -607,212 +606,53 @@ def test_wrong_shape(self): class NonDecoupledTest(tu.TestResultCollector): def setUp(self): self.model_name_ = "repeat_int32" - self.data_matrix = [ - # ("IN", "DELAY", "WAIT") - ([1], [0], [0]), - ([1], [4000], [2000]), - ([1], [2000], [4000]), - ] - - # For grpc async infer test - self.callback_error = None - self.callback_result = None - self.callback_invoked_event = threading.Event() - - def _input_data(self, in_value, delay_value, wait_value): - return { - "IN": np.array(in_value, dtype=np.int32), - "DELAY": np.array(delay_value, dtype=np.uint32), - "WAIT": np.array(wait_value, dtype=np.uint32), + self.input_data = { + "IN": np.array([1], dtype=np.int32), + "DELAY": np.array([0], dtype=np.uint32), + "WAIT": np.array([0], dtype=np.uint32), } - def _async_callback(self, result, error): - """Callback for async_infer.""" - self.callback_error = error - self.callback_result = result - self.callback_invoked_event.set() - def test_grpc(self): - for in_value, delay_value, wait_value in self.data_matrix: - with self.subTest(IN=in_value, DELAY=delay_value, WAIT=wait_value): - input_data = self._input_data(in_value, delay_value, wait_value) - inputs = [ - grpcclient.InferInput("IN", [1], "INT32").set_data_from_numpy( - input_data["IN"] - ), - grpcclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( - input_data["DELAY"] - ), - grpcclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( - input_data["WAIT"] - ), - ] - - triton_client = grpcclient.InferenceServerClient( - url="localhost:8001", verbose=True - ) + inputs = [ + grpcclient.InferInput("IN", [1], "INT32").set_data_from_numpy( + self.input_data["IN"] + ), + grpcclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( + self.input_data["DELAY"] + ), + grpcclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( + self.input_data["WAIT"] + ), + ] - # Expect the inference is successful - res = triton_client.infer(model_name=self.model_name_, inputs=inputs) - self.assertEqual(1, res.as_numpy("OUT")[0]) - self.assertEqual(0, res.as_numpy("IDX")[0]) + triton_client = grpcclient.InferenceServerClient( + url="localhost:8001", verbose=True + ) + # Expect the inference is successful + res = triton_client.infer(model_name=self.model_name_, inputs=inputs) + self.assertEqual(1, res.as_numpy("OUT")[0]) + self.assertEqual(0, res.as_numpy("IDX")[0]) def test_http(self): - for in_value, delay_value, wait_value in self.data_matrix: - with self.subTest(IN=in_value, DELAY=delay_value, WAIT=wait_value): - input_data = self._input_data(in_value, delay_value, wait_value) - inputs = [ - httpclient.InferInput("IN", [1], "INT32").set_data_from_numpy( - input_data["IN"] - ), - httpclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( - input_data["DELAY"] - ), - httpclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( - input_data["WAIT"] - ), - ] - - triton_client = httpclient.InferenceServerClient( - url="localhost:8000", verbose=True - ) - - # Expect the inference is successful - res = triton_client.infer(model_name=self.model_name_, inputs=inputs) - self.assertEqual(1, res.as_numpy("OUT")[0]) - self.assertEqual(0, res.as_numpy("IDX")[0]) - - def test_grpc_async(self): - for in_value, delay_value, wait_value in self.data_matrix: - with self.subTest(IN=in_value, DELAY=delay_value, WAIT=wait_value): - input_data = self._input_data(in_value, delay_value, wait_value) - inputs = [ - grpcclient.InferInput("IN", [1], "INT32").set_data_from_numpy( - input_data["IN"] - ), - grpcclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( - input_data["DELAY"] - ), - grpcclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( - input_data["WAIT"] - ), - ] - - triton_client = grpcclient.InferenceServerClient( - url="localhost:8001", - verbose=True, - ) - - # Clear previous results - self.callback_error = None - self.callback_result = None - self.callback_invoked_event.clear() - - try: - triton_client.async_infer( - model_name=self.model_name_, - inputs=inputs, - callback=self._async_callback, - ) - except Exception as e: - self.fail(f"Failed to initiate async_infer: {e}") - continue - - # Wait for the callback to be invoked, with a timeout - self.assertTrue( - self.callback_invoked_event.wait(timeout=10), - "Callback not invoked within timeout.", - ) - - # Expect the inference is successful - self.assertIsNone( - self.callback_error, f"Inference failed: {self.callback_error}" - ) - self.assertIsNotNone(self.callback_result, "Inference result is None.") - self.assertEqual(1, self.callback_result.as_numpy("OUT")[0]) - self.assertEqual(0, self.callback_result.as_numpy("IDX")[0]) - - # Wait and check server/model health - time.sleep(5) - self.assertTrue(triton_client.is_model_ready(self.model_name_)) - - def test_grpc_async_cancel(self): - data_matrix = [ - # ("IN", "DELAY", "WAIT") - ([1], [4000], [2000]), - ([1], [2000], [4000]), + inputs = [ + httpclient.InferInput("IN", [1], "INT32").set_data_from_numpy( + self.input_data["IN"] + ), + httpclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( + self.input_data["DELAY"] + ), + httpclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( + self.input_data["WAIT"] + ), ] - for in_value, delay_value, wait_value in data_matrix: - with self.subTest(IN=in_value, DELAY=delay_value, WAIT=wait_value): - input_data = self._input_data(in_value, delay_value, wait_value) - inputs = [ - grpcclient.InferInput("IN", [1], "INT32").set_data_from_numpy( - input_data["IN"] - ), - grpcclient.InferInput("DELAY", [1], "UINT32").set_data_from_numpy( - input_data["DELAY"] - ), - grpcclient.InferInput("WAIT", [1], "UINT32").set_data_from_numpy( - input_data["WAIT"] - ), - ] - - triton_client = grpcclient.InferenceServerClient( - url="localhost:8001", - verbose=True, - ) - - # Clear previous results - self.callback_error = None - self.callback_result = None - self.callback_invoked_event.clear() - - request_handle = None - try: - request_handle = triton_client.async_infer( - model_name=self.model_name_, - inputs=inputs, - callback=self._async_callback, - ) - except Exception as e: - self.fail(f"Failed to initiate async_infer: {e}") - continue - - # Allow request to be fully initiated - time.sleep(0.5) - - # Attempt to cancel the request - if request_handle: - try: - request_handle.cancel() - except Exception as e: - self.fail(f"Error calling request_handle.cancel(): {e}") - continue - else: - self.fail("Invalid request_handle, cannot cancel.") - continue - - # Wait for the callback to be invoked - self.assertTrue( - self.callback_invoked_event.wait(timeout=10), - "Callback not invoked within timeout after cancellation.", - ) - - # Expect the inference is failed - self.assertIsInstance( - self.callback_error, - InferenceServerException, - f"Unexpected error type: {type(self.callback_error)}", - ) - self.assertIn( - "StatusCode.CANCELLED", - self.callback_error.status(), - ) - - # Wait and check server/model health - time.sleep(5) - self.assertTrue(triton_client.is_model_ready(self.model_name_)) + triton_client = httpclient.InferenceServerClient( + url="localhost:8000", verbose=True + ) + # Expect the inference is successful + res = triton_client.infer(model_name=self.model_name_, inputs=inputs) + self.assertEqual(1, res.as_numpy("OUT")[0]) + self.assertEqual(0, res.as_numpy("IDX")[0]) if __name__ == "__main__": diff --git a/qa/L0_decoupled/test.sh b/qa/L0_decoupled/test.sh index 839d0131aa..20ca0fffa3 100755 --- a/qa/L0_decoupled/test.sh +++ b/qa/L0_decoupled/test.sh @@ -196,7 +196,7 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** Test NonDecoupledTest Failed\n***" RET=1 else - check_test_results $TEST_RESULT_FILE 4 + check_test_results $TEST_RESULT_FILE 2 if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" diff --git a/qa/L0_dlpack_multi_gpu/test.sh b/qa/L0_dlpack_multi_gpu/test.sh index 0aaf135b5f..d85a089431 100755 --- a/qa/L0_dlpack_multi_gpu/test.sh +++ b/qa/L0_dlpack_multi_gpu/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,22 +41,10 @@ source ../common/util.sh # Uninstall the non CUDA version of PyTorch pip3 uninstall -y torch -pip3 install torch -f https://download.pytorch.org/whl/cu130 +pip3 install torch==2.3.1+cu118 -f https://download.pytorch.org/whl/torch_stable.html # Install CuPy for testing non_blocking compute streams -pip3 install cupy-cuda13x - -if [ ${CUDA_VERSION%%.*} -gt 12 ]; then - curl -L https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvrtc/linux-x86_64/cuda_nvrtc-linux-x86_64-12.9.86-archive.tar.xz \ - -o /tmp/cuda_nvrtc-linux-x86_64-12.9.86-archive.tar.xz ; - curl -L https://developer.download.nvidia.com/compute/cuda/redist/libcublas/linux-x86_64/libcublas-linux-x86_64-12.9.1.4-archive.tar.xz \ - -o /tmp/libcublas-linux-x86_64-12.9.1.4-archive.tar.xz ; - cd /tmp ; - tar -xvf /tmp/cuda_nvrtc-linux-x86_64-12.9.86-archive.tar.xz --strip-components=1 ; - tar -xvf /tmp/libcublas-linux-x86_64-12.9.1.4-archive.tar.xz --strip-components=1 ; - export LD_LIBRARY_PATH=/tmp/lib:$LD_LIBRARY_PATH ; - cd - -fi +pip3 install cupy-cuda12x rm -fr *.log ./models diff --git a/qa/L0_grpc/python_grpc_aio_test.py b/qa/L0_grpc/python_grpc_aio_test.py index 6466432c6b..ba43b36abb 100755 --- a/qa/L0_grpc/python_grpc_aio_test.py +++ b/qa/L0_grpc/python_grpc_aio_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -100,11 +100,9 @@ async def test_get_system_shared_memory_status(self): async def test_register_system_shared_memory(self): with self.assertRaisesRegex( InferenceServerException, - "\[StatusCode\.INTERNAL\] Unable to open shared memory region: '/test_shm'", + "\[StatusCode\.INTERNAL\] Unable to open shared memory region: ''", ): - await self._triton_client.register_system_shared_memory( - "test_shm", "/test_shm", 0 - ) + await self._triton_client.register_system_shared_memory("", "", 0) async def test_unregister_system_shared_memory(self): await self._triton_client.unregister_system_shared_memory() diff --git a/qa/L0_grpc/python_unit_test.py b/qa/L0_grpc/python_unit_test.py index 5ed87078cf..9591d4274c 100755 --- a/qa/L0_grpc/python_unit_test.py +++ b/qa/L0_grpc/python_unit_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,10 +34,7 @@ import numpy as np import tritonclient.grpc as grpcclient -from tritonclient.grpc import service_pb2, service_pb2_grpc -from tritonclient.utils import InferenceServerException, deserialize_bytes_tensor - -import grpc +from tritonclient.utils import InferenceServerException class UserData: @@ -52,90 +49,6 @@ def callback(user_data, result, error): user_data._completed_requests.put(result) -class GrpcTest(unittest.TestCase): - def test_duplicate_output_names_rejected(self): - """Test that duplicate output names in a gRPC infer request are rejected.""" - client = grpcclient.InferenceServerClient(url="localhost:8001") - inputs = [ - grpcclient.InferInput("INPUT0", [1, 16], "INT32"), - grpcclient.InferInput("INPUT1", [1, 16], "INT32"), - ] - inputs[0].set_data_from_numpy(np.ones(shape=(1, 16), dtype=np.int32)) - inputs[1].set_data_from_numpy(np.ones(shape=(1, 16), dtype=np.int32)) - - num_duplicates = 2 - outputs = [ - grpcclient.InferRequestedOutput("OUTPUT0") for _ in range(num_duplicates) - ] - - with self.assertRaises(InferenceServerException) as ctx: - client.infer(model_name="simple", inputs=inputs, outputs=outputs) - self.assertIn( - "output 'OUTPUT0' already exists in request", - str(ctx.exception), - ) - - self.assertTrue( - client.is_server_live(), - "Server is not healthy after duplicate output request", - ) - - def test_bytes_contents_many_elements_serialization(self): - """ - Regression test for InferGRPCToInput bytes_contents pre-allocation. - Sends a BYTES tensor with many explicit bytes_contents elements over - the raw gRPC stub. - """ - # 10,485,760 elements * (4-byte length + 1-byte payload) = 50 MiB - # of serialized BYTES data on both the request and response sides. - element_count = 10 * 1024 * 1024 - payload = b"A" - expected_serialized_size = element_count * (4 + len(payload)) - - request = service_pb2.ModelInferRequest() - request.model_name = "string_identity" - - input_tensor = request.inputs.add() - input_tensor.name = "INPUT0" - input_tensor.datatype = "BYTES" - input_tensor.shape.extend([element_count]) - input_tensor.contents.bytes_contents.extend([payload] * element_count) - request.outputs.add().name = "OUTPUT0" - - # The default Python gRPC client send/receive limit is 4 MiB which - # is below the request and response sizes used here. - channel_options = [ - ("grpc.max_send_message_length", 256 * 1024 * 1024), - ("grpc.max_receive_message_length", 256 * 1024 * 1024), - ] - grpc_stub = service_pb2_grpc.GRPCInferenceServiceStub( - grpc.insecure_channel("localhost:8001", options=channel_options) - ) - response = grpc_stub.ModelInfer(request, timeout=120) - - self.assertEqual(list(response.outputs[0].shape), [element_count]) - self.assertEqual(len(response.raw_output_contents[0]), expected_serialized_size) - - # Verify every element round-tripped correctly through the - # serialization path. - output = deserialize_bytes_tensor(response.raw_output_contents[0]) - self.assertEqual(output.shape, (element_count,)) - self.assertTrue( - np.all(output == payload), - "bytes_contents elements did not round-trip correctly", - ) - - client = grpcclient.InferenceServerClient(url="localhost:8001") - self.assertTrue( - client.is_server_live(), - "Server must remain healthy after many bytes_contents elements", - ) - self.assertTrue( - client.is_model_ready("string_identity"), - "Model must remain ready after many bytes_contents elements", - ) - - class RestrictedProtocolTest(unittest.TestCase): def setUp(self): self.client_ = grpcclient.InferenceServerClient(url="localhost:8001") diff --git a/qa/L0_grpc/test.sh b/qa/L0_grpc/test.sh index c17c3db30c..28d9236d0a 100755 --- a/qa/L0_grpc/test.sh +++ b/qa/L0_grpc/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -152,25 +152,13 @@ sed -i "/CONTROL_SEQUENCE_CORRID/{n;s/data_type:.*/data_type: TYPE_STRING/}" ${M rm -f ${MODELDIR}/simple_string_dyna_sequence/1/model.onnx cp ../custom_models/custom_dyna_sequence_int32/1/libtriton_dyna_sequence.so ${MODELDIR}/simple_string_dyna_sequence/1/ -# Prepare a dedicated model repository for GrpcTest in python_unit_test.py. -# separate from MODELDIR avoids changing the model count expected by other -# tests in this script. -GRPC_TEST_MODELDIR=`pwd`/grpc_test_models -rm -rf ${GRPC_TEST_MODELDIR} -mkdir -p ${GRPC_TEST_MODELDIR} -cp -r ${MODELDIR}/simple ${GRPC_TEST_MODELDIR}/ -cp -r ../python_models/string_identity ${GRPC_TEST_MODELDIR}/string_identity -mkdir -p ${GRPC_TEST_MODELDIR}/string_identity/1 -mv ${GRPC_TEST_MODELDIR}/string_identity/model.py ${GRPC_TEST_MODELDIR}/string_identity/1/model.py -sed -i "s/dims: \[ 1 \]/dims: [ -1 ]/g" ${GRPC_TEST_MODELDIR}/string_identity/config.pbtxt - rm -f *.log rm -f *.log.* set -e CLIENT_LOG=`pwd`/client.log -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR} --allow-client-shm=true" +SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR}" source ../common/util.sh run_server @@ -217,19 +205,19 @@ for i in \ EXTRA_ARGS="-i grpc -u localhost:8001" fi if [[ ($SUFFIX == "image_client") || ($SUFFIX == "grpc_image_client") ]]; then - python $i -m densenet_onnx -s INCEPTION -a -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.async.${SUFFIX} RET=1 fi - python $i -m densenet_onnx -s INCEPTION -a --streaming -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.streaming.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a --streaming -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.streaming.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.streaming.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.streaming.${SUFFIX} RET=1 fi - python $i -m densenet_onnx -s INCEPTION -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.${SUFFIX} @@ -282,19 +270,19 @@ for i in \ BASE=$(basename -- $i) SUFFIX="${BASE%.*}" if [[ $SUFFIX == "image_client" ]]; then - $i -m densenet_onnx -s INCEPTION -a -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.async.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} RET=1 fi - $i -m densenet_onnx -s INCEPTION -a --streaming -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.streaming.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -a --streaming -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.streaming.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.streaming.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} RET=1 fi - $i -m densenet_onnx -s INCEPTION -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} @@ -365,29 +353,6 @@ set -e kill $SERVER_PID wait $SERVER_PID -# Test duplicate output and bytes_contents -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${GRPC_TEST_MODELDIR}" -SERVER_LOG=./inference_server_grpc_test.log -GRPC_TEST_CLIENT_LOG=./inference_client_grpc_test.log -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python $PYTHON_UNIT_TEST GrpcTest >> $GRPC_TEST_CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $GRPC_TEST_CLIENT_LOG - echo -e "\n***\n*** Python GRPC Test Failed\n***" - RET=1 -fi - -set -e -kill $SERVER_PID -wait $SERVER_PID - SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${CLIENT_PLUGIN_MODELDIR} --http-header-forward-pattern=.* --grpc-header-forward-pattern=.*" run_server if [ "$SERVER_PID" == "0" ]; then @@ -604,7 +569,7 @@ done # Run python grpc aio unit test PYTHON_GRPC_AIO_TEST=python_grpc_aio_test.py CLIENT_LOG=`pwd`/python_grpc_aio_test.log -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR} --allow-client-shm=true" +SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR}" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" diff --git a/qa/L0_http/generate_endpoint_test.py b/qa/L0_http/generate_endpoint_test.py index df9317b4dd..3eb0b6ea5f 100755 --- a/qa/L0_http/generate_endpoint_test.py +++ b/qa/L0_http/generate_endpoint_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -29,7 +29,6 @@ sys.path.append("../common") -import base64 import json import threading import time @@ -275,37 +274,6 @@ def test_invalid_input_types(self): self.generate_expect_failure(self._model_name, inputs, error_msg) self.generate_stream_expect_failure(self._model_name, inputs, error_msg) - def test_json_dtype_size_expansion_exceeds_limit_error(self): - """ - Test that when the client sends a JSON input of byte[], that when it - expands to dtype[], it exceeds the maximum allowed input size and - returns an appropriate error message. The test sends a large base64 - encoded string as input, which simulates a byte[] input that would - expand to a much larger dtype[] input on the server side when - `sizeof(dtype) > 1`. - The test checks that the error message indicates that the input size - exceeds the limit. - This is important to prevent clients from sending inputs that could - cause excessive memory usage on the server. - """ - - input_data = [1] * ( - 64 * 1024 * 1024 - ) # 64MB input, which is large but still reasonable for HTTP request body - input_bytes = bytes(input_data) - input_str = base64.b64encode(input_bytes).decode("utf-8") - inputs = {"PROMPT": input_str, "STREAM": False} - error_msg = " bytes exceeds the maximum allowed input size of " - self.generate_expect_failure(self._model_name, inputs, error_msg) - - inputs = { - "INPUT0": input_str[0 : (len(input_str) // 2)], - "INPUT1": input_str[(len(input_str) // 2) :], - "STREAM": False, - } - error_msg = " bytes exceeds the maximum allowed input size of " - self.generate_expect_failure(self._model_name, inputs, error_msg) - def test_duplicate_inputs(self): dupe_prompt = "input 'PROMPT' already exists in request" dupe_stream = "input 'STREAM' already exists in request" diff --git a/qa/L0_http/http_input_size_limit_test.py b/qa/L0_http/http_input_size_limit_test.py deleted file mode 100755 index aed855dd69..0000000000 --- a/qa/L0_http/http_input_size_limit_test.py +++ /dev/null @@ -1,796 +0,0 @@ -#!/usr/bin/python -# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import base64 -import gzip -import io -import json -import unittest - -import numpy as np -import requests -from test_util import GIB, MIB, TestResultCollector, get_server_process_from_env - -# Constants for size calculations -# Each FP32 value is 4 bytes, so we need to divide target byte sizes by 4 to get element counts -BYTES_PER_FP32 = 4 -BYTES_PER_INT64 = ( - 8 # For the type size explosion test, we use int64 which is 8 bytes per element -) -DEFAULT_LIMIT_BYTES = 64 * MIB # 64MB default limit -INCREASED_LIMIT_BYTES = 128 * MIB # 128MB increased limit - -# Calculate element counts for size limits -DEFAULT_LIMIT_ELEMENTS = DEFAULT_LIMIT_BYTES // BYTES_PER_FP32 # 16,777,216 elements -INCREASED_LIMIT_ELEMENTS = ( - INCREASED_LIMIT_BYTES // BYTES_PER_FP32 -) # 33,554,432 elements - -# Small offsets to go just over/under the limits -OFFSET_ELEMENTS = 32 - - -class InferSizeLimitTest(TestResultCollector): - def _get_infer_url(self, model_name): - return f"http://localhost:8000/v2/models/{model_name}/infer" - - def test_json_dtype_size_expansion_exceeds_limit_error(self): - """ - Test that when the client sends a JSON input of byte[], that when it - expands to dtype[], it exceeds the maximum allowed input size and - returns an appropriate error message. The test sends a large base64 - encoded string as input, which simulates a byte[] input that would - expand to a much larger dtype[] input on the server side when - `sizeof(dtype) > 1`. - The test checks that the error message indicates that the input size - exceeds the limit. - This is important to prevent clients from sending inputs that could - cause excessive memory usage on the server. - """ - model = "onnx_zero_1_float32" - - # Provided data is 64MB of int8, but the model expects FP32, - # which would expand to 256MB when interpreted as FP32. - bytes_input = np.ones(DEFAULT_LIMIT_BYTES, dtype=np.int8) - input_bytes = bytes_input.tobytes() - data_str = base64.b64encode(input_bytes).decode("utf-8") - headers = { - "Content-Type": "application/json", - "Inference-Header-Content-Length": f"{len(input_bytes)}", - } - shape_size = ( - DEFAULT_LIMIT_ELEMENTS // BYTES_PER_INT64 - ) # Calculate shape size based on int64 element count to match the byte size - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "INT64", - "shape": [1, shape_size], - "data": data_str, - } - ] - } - - response = requests.post( - f"http://localhost:8000/v2/models/{model}/generate", - headers=headers, - json=payload, - ) - - self.assertEqual( - 400, - response.status_code, - f"Expected error code for type/size mismatch, got: {response.status_code}", - ) - error_msg = response.content.decode() - print( - f"Error message: {error_msg}", flush=True - ) # Print the error message for debugging - self.assertIn( - "request JSON size of ", - error_msg, - ) - self.assertIn( - " bytes exceeds the maximum allowed input size of ", - error_msg, - ) - self.assertIn( - "Use --http-max-input-size to increase the limit.", - error_msg, - ) - - # Test multiple inputs with one that causes size explosion. - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "INT64", - "shape": [1, shape_size // 2], - "data": data_str[: len(data_str) // 2], - }, - { - "name": "INPUT1", - "datatype": "INT64", - "shape": [1, shape_size // 2], - "data": data_str[len(data_str) // 2 :], - }, - ] - } - - response = requests.post( - f"http://localhost:8000/v2/models/{model}/generate", - headers=headers, - json=payload, - ) - - self.assertEqual( - 400, - response.status_code, - f"Expected error code for type/size mismatch, got: {response.status_code}", - ) - error_msg = response.content.decode() - print( - f"Error message: {error_msg}", flush=True - ) # Print the error message for debugging - self.assertIn( - "request JSON size of ", - error_msg, - ) - self.assertIn( - " bytes exceeds the maximum allowed input size of ", - error_msg, - ) - self.assertIn( - "Use --http-max-input-size to increase the limit.", - error_msg, - ) - - def test_default_limit_raw_binary(self): - """Test raw binary inputs with default limit""" - model = "onnx_zero_1_float32" - - # Test case 1: Input just over the 64MB limit (should fail) - # (2^24 + 32) elements * 4 bytes = 64MB + 128 bytes = 67,108,992 bytes - large_input = np.ones( - DEFAULT_LIMIT_ELEMENTS + OFFSET_ELEMENTS, dtype=np.float32 - ) - input_bytes = large_input.tobytes() - assert len(input_bytes) > 64 * MIB # Verify we're actually over the 64MB limit - - headers = {"Inference-Header-Content-Length": "0"} - response = requests.post( - self._get_infer_url(model), data=input_bytes, headers=headers - ) - - # Should fail with 400 bad request with default limit - self.assertEqual( - 400, - response.status_code, - "Expected error code for oversized request, got: {}".format( - response.status_code - ), - ) - - # Verify error message contains size limit info - error_msg = response.content.decode() - self.assertIn( - "exceeds the maximum allowed value", - error_msg, - "Expected error message about exceeding max input size", - ) - - # Test case 2: Input just under the 64MB limit (should succeed) - # (2^24 - 32) elements * 4 bytes = 64MB - 128 bytes = 67,108,736 bytes - small_input = np.ones( - DEFAULT_LIMIT_ELEMENTS - OFFSET_ELEMENTS, dtype=np.float32 - ) - input_bytes = small_input.tobytes() - assert len(input_bytes) < 64 * MIB # Verify we're actually under the 64MB limit - - response = requests.post( - self._get_infer_url(model), data=input_bytes, headers=headers - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - "Expected success code for request within size limit, got: {}".format( - response.status_code - ), - ) - - # Verify output matches our input (identity model) - header_size = int(response.headers["Inference-Header-Content-Length"]) - output_data = response.content[header_size:] - - # Convert output bytes back to numpy array for comparison - output_array = np.frombuffer(output_data, dtype=np.float32) - self.assertTrue( - np.array_equal(output_array, small_input), - "Response data does not match input data", - ) - - def test_default_limit_json(self): - """Test JSON inputs with default limit""" - model = "onnx_zero_1_float32" - - # Test case 1: Input just over the 64MB limit (should fail) - # (2^24 + 32) elements * 4 bytes = 64MB + 128 bytes = 67,108,992 bytes - shape_size = DEFAULT_LIMIT_ELEMENTS + OFFSET_ELEMENTS - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, shape_size], - "data": [1.0] * shape_size, - } - ] - } - assert ( - shape_size * BYTES_PER_FP32 > 64 * MIB - ) # Verify we're actually over the 64MB limit - - headers = {"Content-Type": "application/json"} - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - # Should fail with 400 bad request with default limit - self.assertEqual( - 400, - response.status_code, - "Expected error code for oversized JSON request, got: {}".format( - response.status_code - ), - ) - - # Verify error message contains size limit info - error_msg = response.content.decode() - self.assertIn( - "request JSON size of ", - error_msg, - ) - self.assertIn( - " bytes exceeds the maximum allowed input size of ", - error_msg, - ) - self.assertIn( - "Use --http-max-input-size to increase the limit.", - error_msg, - ) - - # Test case 2: Input just under the 64MB limit (should succeed) - # The test creates a JSON payload with data, which adds overhead compared - # to raw binary format. We adjust the shape size to ensure the final - # JSON payload is under the size limit. An element is roughly 5 - # bytes in JSON, compared to 4 bytes as a raw FP32. - shape_size = (DEFAULT_LIMIT_ELEMENTS - OFFSET_ELEMENTS) * 4 // 5 - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, shape_size], - "data": [1.0] * shape_size, - } - ] - } - # Verify we're actually under the 64MB limit - self.assertLess(len(json.dumps(payload).encode("utf-8")), DEFAULT_LIMIT_BYTES) - - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - "Expected success code for JSON request within size limit, got: {}".format( - response.status_code - ), - ) - - # Verify we got a valid response - result = response.json() - self.assertIn("outputs", result, "Response missing outputs field") - self.assertEqual(1, len(result["outputs"]), "Expected 1 output") - self.assertEqual( - shape_size, - result["outputs"][0]["shape"][1], - f"Expected shape {[1, shape_size]}, got {result['outputs'][0]['shape']}", - ) - - def test_large_input_raw_binary(self): - """Test raw binary input larger with custom limit set""" - model = "onnx_zero_1_float32" - - # Test case 1: Input just over the 128MB configured limit (should fail) - # (2^25 + 32) elements * 4 bytes = 128MB + 128 bytes = 134,217,856 bytes - large_input = np.ones( - INCREASED_LIMIT_ELEMENTS + OFFSET_ELEMENTS, dtype=np.float32 - ) - input_bytes = large_input.tobytes() - assert ( - len(input_bytes) > 128 * MIB - ) # Verify we're actually over the 128MB limit - - headers = {"Inference-Header-Content-Length": "0"} - response = requests.post( - self._get_infer_url(model), data=input_bytes, headers=headers - ) - - # Should fail with 400 bad request with our increased limit - self.assertEqual( - 400, - response.status_code, - "Expected error code for oversized request, got: {}".format( - response.status_code - ), - ) - - # Verify error message contains size limit info - error_msg = response.content.decode() - self.assertIn( - "exceeds the maximum allowed value", - error_msg, - "Expected error message about exceeding max input size", - ) - - # Test case 2: Input just under the 128MB configured limit (should succeed) - # (2^25 - 32) elements * 4 bytes = 128MB - 128 bytes = 134,217,600 bytes - small_input = np.ones( - INCREASED_LIMIT_ELEMENTS - OFFSET_ELEMENTS, dtype=np.float32 - ) - input_bytes = small_input.tobytes() - assert ( - len(input_bytes) < 128 * MIB - ) # Verify we're actually under the 128MB limit - - response = requests.post( - self._get_infer_url(model), data=input_bytes, headers=headers - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - "Expected success code for request within increased limit, got: {}".format( - response.status_code - ), - ) - - # Verify output matches our input (identity model) - header_size = int(response.headers["Inference-Header-Content-Length"]) - output_data = response.content[header_size:] - - # Convert output bytes back to numpy array for comparison - output_array = np.frombuffer(output_data, dtype=np.float32) - self.assertTrue( - np.array_equal(output_array, small_input), - "Response data does not match input data", - ) - - def test_large_input_json(self): - """Test JSON input larger with custom limit set""" - model = "onnx_zero_1_float32" - - # Test case 1: Input just over the 128MB configured limit (should fail) - # (2^25 + 32) elements * 4 bytes = 128MB + 128 bytes = 134,217,856 bytes - shape_size = INCREASED_LIMIT_ELEMENTS + OFFSET_ELEMENTS - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, shape_size], - "data": [1.0] * shape_size, - } - ] - } - assert ( - shape_size * BYTES_PER_FP32 > 128 * MIB - ) # Verify we're actually over the 128MB limit - - headers = {"Content-Type": "application/json"} - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - # Should fail with 400 bad request with our increased limit - self.assertEqual( - 400, - response.status_code, - "Expected error code for oversized JSON request, got: {}".format( - response.status_code - ), - ) - - # Verify error message contains size limit info - error_msg = response.content.decode() - self.assertIn( - "request JSON size of ", - error_msg, - ) - self.assertIn( - " bytes exceeds the maximum allowed input size of ", - error_msg, - ) - self.assertIn( - "Use --http-max-input-size to increase the limit.", - error_msg, - ) - - # Test case 2: Input just under the 128MB configured limit (should succeed) - # The test creates a JSON payload with data, which adds overhead compared - # to raw binary format. We adjust the shape size to ensure the final - # JSON payload is under the size limit. An element is roughly 5 - # bytes in JSON, compared to 4 bytes as a raw FP32. - shape_size = (INCREASED_LIMIT_ELEMENTS - OFFSET_ELEMENTS) * 4 // 5 - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, shape_size], - "data": [1.0] * shape_size, - } - ] - } - # Verify we're actually under the 128MB limit - self.assertLess(len(json.dumps(payload).encode("utf-8")), INCREASED_LIMIT_BYTES) - - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - "Expected success code for request within increased limit, got: {}".format( - response.status_code - ), - ) - - # Verify we got a valid response - result = response.json() - self.assertIn("outputs", result, "Response missing outputs field") - self.assertEqual(1, len(result["outputs"]), "Expected 1 output") - self.assertEqual( - shape_size, - result["outputs"][0]["shape"][1], - f"Expected shape {[1, shape_size]}, got {result['outputs'][0]['shape']}", - ) - - def test_large_string_in_json(self): - """Test JSON request with large string input""" - model = "simple_identity" - - # Create a string that is larger (large payload about 2GB) than the default limit of 64MB - # (2^31 + 64) elements * 1 bytes = 2GB + 64 bytes = 2,147,483,712 bytes - large_string_size = 2 * GIB + 64 - large_string = "A" * large_string_size - - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "BYTES", - "shape": [1, 1], - "data": [large_string], - } - ] - } - - headers = {"Content-Type": "application/json"} - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - # Should fail with 400 bad request - self.assertEqual( - 400, - response.status_code, - "Expected error code for oversized JSON request, got: {}".format( - response.status_code - ), - ) - - # Verify error message - error_msg = response.content.decode() - self.assertIn( - "request JSON size of ", - error_msg, - ) - self.assertIn( - " bytes exceeds the maximum allowed input size of ", - error_msg, - ) - self.assertIn( - "Use --http-max-input-size to increase the limit.", - error_msg, - ) - - def _create_compressed_payload(self, target_size): - """Helper to create a gzip-compressed JSON payload of specified decompressed size.""" - shape_size = 1000 # Small actual data - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, shape_size], - "data": [1.0] * shape_size, - } - ] - } - json_str = json.dumps(payload, indent=4) - - # Pad with whitespace to reach target size (whitespace before closing brace is valid JSON) - padding_needed = target_size - len(json_str) - padded_json = json_str[:-1] + (" " * padding_needed) + json_str[-1] - - # Compress the payload - compressed_buffer = io.BytesIO() - with gzip.GzipFile(fileobj=compressed_buffer, mode="wb") as gz: - gz.write(padded_json.encode("utf-8")) - - return compressed_buffer.getvalue(), len(padded_json.encode("utf-8")) - - def test_default_limit_compressed(self): - """Test compressed inputs with default 64MB limit. - - This test verifies that the --http-max-input-size limit is enforced on - the decompressed data size, not just the compressed request size. - """ - model = "onnx_zero_1_float32" - - headers = { - "Content-Type": "application/json", - "Content-Encoding": "gzip", - } - - # Test case 1: Payload that decompresses to 64MB + 1MB (over limit) should fail - large_target_size = DEFAULT_LIMIT_BYTES + MIB - ( - large_compressed_data, - large_uncompressed_size, - ) = self._create_compressed_payload(large_target_size) - - # Verify uncompressed size is over 64MB limit - self.assertGreater( - large_uncompressed_size, - DEFAULT_LIMIT_BYTES, - f"Large payload should decompress to > 64MB, got {large_uncompressed_size}", - ) - - # Verify compressed size is under the limit - self.assertLess( - len(large_compressed_data), - DEFAULT_LIMIT_BYTES, - f"Compressed size should be under limit, got {len(large_compressed_data)}", - ) - - response = requests.post( - self._get_infer_url(model), data=large_compressed_data, headers=headers - ) - - # Should fail with 400 bad request - decompressed size exceeds limit - self.assertEqual( - 400, - response.status_code, - f"Expected 400 for compressed request that decompresses to >64MB, got: {response.status_code}", - ) - - # Verify error message contains size limit info - error_msg = response.content.decode() - self.assertIn( - "exceeds the maximum allowed value", - error_msg, - "Expected error message about exceeding max input size", - ) - - # Test case 2: Payload that decompresses to 64MB - 1MB (under limit) should succeed - small_target_size = DEFAULT_LIMIT_BYTES - MIB - ( - small_compressed_data, - small_uncompressed_size, - ) = self._create_compressed_payload(small_target_size) - - # Verify uncompressed size is under 64MB limit - self.assertLess( - small_uncompressed_size, - DEFAULT_LIMIT_BYTES, - f"Small payload should decompress to < 64MB, got {small_uncompressed_size}", - ) - - response = requests.post( - self._get_infer_url(model), data=small_compressed_data, headers=headers - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - f"Expected 200 for compressed request within limit, got: {response.status_code}", - ) - - # Verify we got a valid response - result = response.json() - self.assertIn("outputs", result, "Response missing outputs field") - - def test_large_input_compressed(self): - """Test compressed inputs with custom 128MB limit set. - - This test verifies that compressed inputs work correctly when the - --http-max-input-size limit is increased. - """ - model = "onnx_zero_1_float32" - - headers = { - "Content-Type": "application/json", - "Content-Encoding": "gzip", - } - - # Test case 1: Input that decompresses to 128MB + 1MB (over limit) should fail - large_target_size = INCREASED_LIMIT_BYTES + MIB - ( - large_compressed_data, - large_uncompressed_size, - ) = self._create_compressed_payload(large_target_size) - - # Verify sizes - self.assertGreater( - large_uncompressed_size, - INCREASED_LIMIT_BYTES, - f"Large payload should decompress to > 128MB, got {large_uncompressed_size}", - ) - - response = requests.post( - self._get_infer_url(model), data=large_compressed_data, headers=headers - ) - - # Should fail with 400 bad request - self.assertEqual( - 400, - response.status_code, - f"Expected 400 for compressed request exceeding 128MB limit, got: {response.status_code}", - ) - - error_msg = response.content.decode() - self.assertIn( - "exceeds the maximum allowed value", - error_msg, - "Expected error message about exceeding max input size", - ) - - # Test case 2: Input that decompresses to 128MB - 1MB (under limit) should succeed - small_target_size = INCREASED_LIMIT_BYTES - MIB - ( - small_compressed_data, - small_uncompressed_size, - ) = self._create_compressed_payload(small_target_size) - - # Verify sizes - self.assertLess( - small_uncompressed_size, - INCREASED_LIMIT_BYTES, - f"Small payload should decompress to < 128MB, got {small_uncompressed_size}", - ) - self.assertGreater( - small_uncompressed_size, - DEFAULT_LIMIT_BYTES, - f"Small payload should decompress to > 64MB (default), got {small_uncompressed_size}", - ) - - response = requests.post( - self._get_infer_url(model), data=small_compressed_data, headers=headers - ) - - # Should succeed with 200 OK - self.assertEqual( - 200, - response.status_code, - f"Expected 200 for compressed request within 128MB limit, got: {response.status_code}", - ) - - # Verify we got a valid response - result = response.json() - self.assertIn("outputs", result, "Response missing outputs field") - - def test_no_leak_on_invalid_inference_header_length(self): - """ - Test that sending multiple malformed compressed requests does not cause memory growth on the server. - """ - leak_request_count = 100 - max_rss_growth_bytes = 32 * MIB - model = "onnx_zero_1_float32" - - body = gzip.compress(b" " * MIB) - headers = { - "Content-Type": "application/json", - "Content-Encoding": "gzip", - "Inference-Header-Content-Length": "9999999", - } - url = self._get_infer_url(model) - - server = get_server_process_from_env() - - with requests.Session() as session: - # Warm up the failure path so one-time allocations do not look - # like leaks. - resp = session.post(url, data=body, headers=headers) - self.assertEqual( - 400, - resp.status_code, - f"Expected status code 400, got {resp.status_code}: " - f"{resp.content[:200]!r}", - ) - - rss_before = server.memory_info().rss - for _ in range(leak_request_count): - resp = session.post(url, data=body, headers=headers) - self.assertEqual( - 400, - resp.status_code, - f"Expected status code 400, got {resp.status_code}: " - f"{resp.content[:200]!r}", - ) - rss_after = server.memory_info().rss - - growth = rss_after - rss_before - print( - f"RSS: before={rss_before / MIB:.1f} MIB, " - f"after={rss_after / MIB:.1f} MIB, " - f"growth={growth / MIB:.1f} MIB, " - f"limit={max_rss_growth_bytes / MIB:.0f} MIB", - flush=True, - ) - self.assertLess( - growth, - max_rss_growth_bytes, - f"Server RSS grew by {growth / MIB:.1f} MIB after " - f"{leak_request_count} malformed compressed requests " - f"(limit {max_rss_growth_bytes / MIB:.0f} MIB).", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_http/http_request_many_chunks.py b/qa/L0_http/http_request_many_chunks.py deleted file mode 100755 index d733fb000c..0000000000 --- a/qa/L0_http/http_request_many_chunks.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/python -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import socket -import sys -import unittest - -sys.path.append("../common") -from test_util import MIB, get_server_process_from_env, wait_for_stable_rss - - -class HTTPRequestManyChunksTest(unittest.TestCase): - def setUp(self): - self._local_host = "localhost" - self._http_port = 8000 - self._model_name = "simple" - # Must match server kMaxChunkedChunks (http_server.cc). - self._k_max_chunked_chunks = 65536 - self._over_max_chunks_error = f"Chunked request body exceeds maximum of {self._k_max_chunked_chunks} non-empty chunks. Send fewer or larger HTTP chunks." - - def _infer_chunked_header(self): - return ( - f"POST /v2/models/{self._model_name}/infer HTTP/1.1\r\n" - f"Inference-Header-Content-Length: 0\r\n" - ) - - def send_chunked_request( - self, - header: str, - chunk_count: int, - expected_response: str, - expected_http_status=400, - ): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - header = ( - f"{header}" - f"Host: {self._local_host}:{self._http_port}\r\n" - f"Content-Type: application/octet-stream\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ) - try: - s.connect((self._local_host, self._http_port)) - # HTTP request with chunked encoding - s.sendall((header.encode())) - - # Send chunked payload - for _ in range(chunk_count): - try: - s.send(b"1\r\nA\r\n") - except (BrokenPipeError, ConnectionResetError): - break - try: - s.sendall(b"0\r\n\r\n") - except (BrokenPipeError, ConnectionResetError): - # Server may close/reset early after deciding on an error response. - # Ignore send failure here and continue reading any available response bytes. - pass - - response = b"" - while True: - try: - chunk = s.recv(4096) - if not chunk: - break - response += chunk - except ConnectionResetError: - break - except socket.timeout: - break - self.assertTrue( - response, - "expected error response body, but socket closed/reset before any bytes", - ) - status_line = response.split(b"\r\n", 1)[0].decode(errors="replace") - self.assertTrue( - status_line.startswith(f"HTTP/1.1 {expected_http_status} "), - f"expected HTTP status {expected_http_status}, got {status_line!r}", - ) - self.assertIn(expected_response, response.decode()) - except Exception as e: - raise (e) - finally: - s.close() - - def test_chunked_infer_at_max_chunks(self): - """Exactly kMaxChunkedChunks non-empty chunks: 400 request input error.""" - self.send_chunked_request( - self._infer_chunked_header(), - self._k_max_chunked_chunks, - "Raw request must only have 1 input (found 1) to be deduced but got 2 " - "inputs in 'simple' model configuration", - ) - - def test_chunked_infer_rejected_over_max_chunks(self): - """kMaxChunkedChunks+1 chunks: 400 request error with bounded RSS growth.""" - - # Warm up failure path to avoid one-time allocation noise in RSS checks. - self.send_chunked_request( - self._infer_chunked_header(), - self._k_max_chunked_chunks + 1, - self._over_max_chunks_error, - ) - - def test_chunked_infer_over_max_chunks_reject_with_bounded_rss_growth(self): - many_chunks = 1000000 - - # verify server is running - server = get_server_process_from_env("SERVER_PID") - self.assertTrue(server.is_running()) - - # warm up and wait until RSS is stable. - self.send_chunked_request( - self._infer_chunked_header(), - many_chunks, # way over max chunks - self._over_max_chunks_error, - ) - # Wait until RSS is stable across several measurements before continuing. - server = get_server_process_from_env("SERVER_PID") - wait_for_stable_rss(server) - - # Monitor RSS growth over 100 requests. - repeat_request_count = 100 - rss_before = server.memory_info().rss - max_rss_growth_bytes = 1 * MIB - - for _ in range(repeat_request_count): - self.send_chunked_request( - self._infer_chunked_header(), - many_chunks, # way over max chunks - self._over_max_chunks_error, - ) - - rss_after = server.memory_info().rss - growth = rss_after - rss_before - print( - f"RSS: before={rss_before / MIB:.1f} MiB, " - f"after={rss_after / MIB:.1f} MiB, " - f"growth={growth / MIB:.1f} MiB, " - f"limit={max_rss_growth_bytes / MIB:.0f} MiB", - flush=True, - ) - self.assertLess( - growth, - max_rss_growth_bytes, - f"Server RSS grew by {growth / MIB:.1f} MiB after " - f"{repeat_request_count} over-limit chunked infer requests " - f"(limit {max_rss_growth_bytes / MIB:.0f} MiB).", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_http/http_test.py b/qa/L0_http/http_test.py index 9ea65fddf6..638ccbbbf8 100755 --- a/qa/L0_http/http_test.py +++ b/qa/L0_http/http_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -236,104 +236,6 @@ def test_descriptive_status_code(self): ) t.join() - def test_buffer_size_overflow(self): - model = "onnx_zero_1_float32" - - # Test for overflow within GetElementCount() - payload1 = { - "inputs": [ - { - "name": "INPUT0", - "shape": [ - 2**4, - 2**60 + 2, - ], # This evaluates to 2^64 + 32 during GetElementCount() - "datatype": "FP32", - "data": [1.0], - } - ] - } - - # Test for overflow with type_byte_size multiplication - payload2 = { - "inputs": [ - { - "name": "INPUT0", - "shape": [ - 2**2, - 2**60 + 2, - ], # This evaluates to 2^64 + 32 during type_byte_size multiplication since FP32 is 4 bytes - "datatype": "FP32", - "data": [1.0], - } - ] - } - - # Send request and expect a 400 error with specific overflow message - headers = {"Content-Type": "application/json"} - - # Test the first payload (GetElementCount overflow) - r1 = requests.post(self._get_infer_url(model), json=payload1, headers=headers) - - self.assertEqual( - 400, - r1.status_code, - "Expected error code 400 for GetElementCount overflow check; got: {}".format( - r1.status_code - ), - ) - - error_message1 = r1.content.decode() - self.assertIn( - "causes total element count to exceed maximum size of", error_message1 - ) - - # Test the second payload (type_byte_size multiplication overflow) - r2 = requests.post(self._get_infer_url(model), json=payload2, headers=headers) - - self.assertEqual( - 400, - r2.status_code, - "Expected error code 400 for type_byte_size multiplication overflow check; got: {}".format( - r2.status_code - ), - ) - - error_message2 = r2.content.decode() - self.assertIn("byte size overflow for input", error_message2) - - def test_negative_dimensions(self): - model = "onnx_zero_1_float32" - - payload = { - "inputs": [ - { - "name": "INPUT0", - "shape": [2, -5], # Negative dimension should be invalid - "datatype": "FP32", - "data": [1.0], - } - ] - } - - # Send request and expect a 500 error - headers = {"Content-Type": "application/json"} - r = requests.post(self._get_infer_url(model), json=payload, headers=headers) - - self.assertEqual( - 500, - r.status_code, - "Expected error code 500 for negative dimension; got: {}".format( - r.status_code - ), - ) - - error_message = r.content.decode() - self.assertIn( - "Unable to parse 'shape': attempt to access JSON non-unsigned-integer as unsigned-integer", - error_message, - ) - def test_loading_large_invalid_model(self): # Generate large base64 encoded data data_length = 1 << 31 @@ -364,173 +266,13 @@ def test_loading_large_invalid_model(self): try: error_message = response.json().get("error", "") self.assertIn( - "request JSON size", - error_message, - ) - self.assertIn( - " exceeds the maximum allowed input size", + "'file:1/model.onnx' exceeds the maximum allowed data size limit " + "INT_MAX", error_message, ) except ValueError: self.fail("Response is not valid JSON") - def test_load_oversized_file_parameter(self): - # Single path component longer than NAME_MAX (255 on typical Linux) must - # be rejected without terminating the server (filesystem_error handled). - long_path = "file:" + ("A" * 256) - payload = { - "parameters": { - long_path: "YQ==", - "config": "{}", - } - } - headers = {"Content-Type": "application/json"} - response = requests.post( - self._get_load_model_url("onnx_zero_1_float32"), - headers=headers, - json=payload, - ) - # TODO: [TRI-958] Status code 400 is more appropriate here - self.assertEqual( - 500, - response.status_code, - "Expected 500 for oversized file parameter; got {}".format( - response.status_code - ), - ) - try: - self.assertIn( - "failed to poll from model repository", response.json().get("error", "") - ) - except ValueError: - self.fail("Response is not valid JSON") - health = requests.get("http://localhost:8000/v2/health/ready", timeout=10) - self.assertEqual( - 200, - health.status_code, - "server must stay up after rejected load", - ) - - def test_json_recursion_depth_limit(self): - """Test that server properly handles and rejects deeply nested JSON.""" - - def create_nested_json(depth, value): - for _ in range(depth): - value = f"[{value}]" - return json.loads(value) - - headers = {"Content-Type": "application/json"} - test_matrix = [ - # (datatype, data, model, json_depth, should_succeed) - ("BYTES", '"hello"', "simple_identity", 120, False), - ("BYTES", '"hello"', "simple_identity", 50, True), - ("INT64", "123", "simple_identity_int64", 120, False), - ("INT64", "123", "simple_identity_int64", 50, True), - ] - - for dtype, data, model, json_depth, should_succeed in test_matrix: - with self.subTest( - datatype=dtype, depth=json_depth, should_succeed=should_succeed - ): - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": dtype, - "shape": [1, 1], - "data": create_nested_json(json_depth, data), - } - ] - } - - response = requests.post( - self._get_infer_url(model), headers=headers, json=payload - ) - - if should_succeed: - self.assertEqual(response.status_code, 200) - else: - self.assertNotEqual(response.status_code, 200) - try: - error_message = response.json().get("error", "") - self.assertIn( - "JSON nesting depth exceeds maximum allowed limit (100)", - error_message, - ) - except ValueError: - self.fail("Response is not valid JSON") - - def test_duplicate_output_names(self): - """Test that duplicate output names are rejected""" - model = "onnx_zero_1_float32" - input_data = np.arange(8, dtype=np.float32).flatten().tolist() - - num_duplicates = 2 - payload = { - "inputs": [ - { - "name": "INPUT0", - "datatype": "FP32", - "shape": [1, 8], - "data": [input_data], - } - ], - "outputs": [{"name": "OUTPUT0"} for _ in range(num_duplicates)], - } - - headers = {"Content-Type": "application/json"} - r = requests.post(self._get_infer_url(model), json=payload, headers=headers) - self.assertEqual( - 400, - r.status_code, - "Expected error code 400 for duplicate output names; got: {}".format( - r.status_code - ), - ) - error_message = r.json().get("error", "") - self.assertIn("output 'OUTPUT0' already exists in request", error_message) - - # Verify server is still healthy after the bad request - health_url = "http://localhost:8000/v2/health/live" - health_r = requests.get(health_url) - self.assertEqual( - 200, - health_r.status_code, - "Server is not healthy after duplicate output request", - ) - - def test_repository_index_deeply_nested_json(self): - """Test for deeply nested JSON on model repository index.""" - depth = 250000 - nested = ("[" * depth) + "true" + ("]" * depth) - payload = '{"ready":' + nested + "}" - - # Keep request below default --http-max-input-size so parsing path is exercised. - self.assertLess(len(payload), 64 * 1024 * 1024) - - response = requests.post( - "http://localhost:8000/v2/repository/index", - data=payload, - headers={"Content-Type": "application/json"}, - timeout=60, - ) - self.assertEqual( - 400, - response.status_code, - "Expected repository index request to fail on invalid 'ready' type.", - ) - self.assertIn( - "Invalid value for 'ready': expected a boolean", - response.json()["error"], - ) - - live_response = requests.get("http://localhost:8000/v2/health/live", timeout=10) - self.assertEqual( - 200, - live_response.status_code, - "Expected server to remain live after deeply nested JSON request.", - ) - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index e6b48272d2..c36024e007 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -40,7 +40,6 @@ fi export CUDA_VISIBLE_DEVICES=0 -source ../common/util.sh RET=0 CLIENT_PLUGIN_TEST="./http_client_plugin_test.py" @@ -129,7 +128,8 @@ rm -f *.log.* set -e CLIENT_LOG=`pwd`/client.log -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR} --allow-client-shm=true" +SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR}" +source ../common/util.sh run_server if [ "$SERVER_PID" == "0" ]; then @@ -163,13 +163,13 @@ for i in \ BASE=$(basename -- $i) SUFFIX="${BASE%.*}" if [ $SUFFIX == "image_client" ]; then - python $i -m densenet_onnx -s INCEPTION -a -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.async.${SUFFIX} RET=1 fi - python $i -m densenet_onnx -s INCEPTION -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.${SUFFIX} @@ -255,7 +255,7 @@ echo -n 'username:' > pswd echo "password" | openssl passwd -stdin -apr1 >> pswd nginx -c `pwd`/$NGINX_CONF -python $BASIC_AUTH_TEST >> ${CLIENT_LOG}.python.plugin.auth 2>&1 +python $BASIC_AUTH_TEST if [ $? -ne 0 ]; then cat ${CLIENT_LOG}.python.plugin.auth RET=1 @@ -380,15 +380,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -403,15 +397,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -426,15 +414,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -449,15 +431,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "0" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] found in output when not expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "0" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] found in output when not expected" - cat ./curl.out - echo "" RET=1 fi @@ -473,15 +449,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -495,10 +465,7 @@ if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" RET=1 fi -if [ `grep -c "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\}" ./curl.out` != "1" ]; then - echo -e "\{\"error\":\"Failed to parse 'data' field: shape does not match true shape\"\} not found in output when expected" - cat ./curl.out - echo "" +if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then RET=1 fi @@ -512,9 +479,6 @@ if [ "$code" == "200" ]; then RET=1 fi if [ `grep -c "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\}" ./curl.out` != "1" ]; then - echo -e "\{\"error\":\"Unable to parse 'data': Shape does not match true shape of 'data' field\"\} not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -529,15 +493,9 @@ if [ "$code" != "200" ]; then RET=1 fi if [ `grep -c "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\]" ./curl.out` != "1" ]; then - echo -e "\[2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi if [ `grep -c "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\]" ./curl.out` != "1" ]; then - echo -e "\[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0\] not found in output when expected" - cat ./curl.out - echo "" RET=1 fi @@ -612,7 +570,7 @@ done # Run python http aio unit test PYTHON_HTTP_AIO_TEST=python_http_aio_test.py CLIENT_LOG=`pwd`/python_http_aio_test.log -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR} --allow-client-shm=true" +SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR}" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -654,13 +612,7 @@ cp -r ${MODELDIR}/onnx_zero_1_float32 ${MODELDIR}/onnx_zero_1_float32_queue && \ echo " }" >> config.pbtxt && \ echo "}" >> config.pbtxt) -cp -r ./models/simple_identity ${MODELDIR} -cp -r ./models/simple_identity ${MODELDIR}/simple_identity_int64 && \ - (cd $MODELDIR/simple_identity_int64 && \ - sed -i "s/TYPE_STRING/TYPE_INT64/" config.pbtxt && \ - sed -i "s/simple_identity/simple_identity_int64/" config.pbtxt) - -SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR} --model-control-mode=explicit --load-model=*" +SERVER_ARGS="--backend-directory=${BACKEND_DIR} --model-repository=${MODELDIR}" SERVER_LOG="./inference_server_http_test.log" CLIENT_LOG="./http_test.log" run_server @@ -672,7 +624,7 @@ fi TEST_RESULT_FILE='test_results.txt' PYTHON_TEST=http_test.py -EXPECTED_NUM_TESTS=16 +EXPECTED_NUM_TESTS=10 set +e python $PYTHON_TEST >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then @@ -695,9 +647,9 @@ wait $SERVER_PID # Helper library to parse SSE events # https://github.com/mpetazzoni/sseclient -pip install sseclient-py psutil +pip install sseclient-py -SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models --log-verbose=1" +SERVER_ARGS="--model-repository=`pwd`/../python_models/generate_models" SERVER_LOG="./inference_server_generate_endpoint_test.log" CLIENT_LOG="./generate_endpoint_test.log" run_server @@ -710,9 +662,9 @@ fi ## Python Unit Tests TEST_RESULT_FILE='test_results.txt' PYTHON_TEST=generate_endpoint_test.py -EXPECTED_NUM_TESTS=18 +EXPECTED_NUM_TESTS=17 set +e -python $PYTHON_TEST > $CLIENT_LOG 2>&1 +python $PYTHON_TEST >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG RET=1 @@ -795,172 +747,7 @@ set -e kill $SERVER_PID wait $SERVER_PID -### Test HTTP input size limits ### - -# Setup models needed for the test -MODELDIR=http_input_size_limit_test_models -mkdir -p $MODELDIR -rm -rf ${MODELDIR}/* -cp -r $DATADIR/qa_identity_model_repository/onnx_zero_1_float32 ${MODELDIR}/. -cp -r ./models/simple_identity ${MODELDIR}/. - -# First run with default size limit - large inputs should fail -SERVER_ARGS="--model-repository=${MODELDIR}" -SERVER_LOG="./inference_server_default_limit.log" -CLIENT_LOG="./http_input_size_limit_default.log" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -# Run test to verify that large inputs fail with default limit -python http_input_size_limit_test.py InferSizeLimitTest.test_default_limit_raw_binary >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Default Input Size Limit Test Failed for raw binary input\n***" - RET=1 -fi - -python http_input_size_limit_test.py InferSizeLimitTest.test_default_limit_json >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Default Input Size Limit Test Failed for JSON input\n***" - RET=1 -fi - -python http_input_size_limit_test.py InferSizeLimitTest.test_large_string_in_json >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Default Input Size Limit Test Failed for large string in JSON\n***" - RET=1 -fi - -python http_input_size_limit_test.py InferSizeLimitTest.test_default_limit_compressed >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Default Input Size Limit Test Failed for compressed input\n***" - RET=1 -fi - -# Run test to verify that large inputs fail with default limit -python http_input_size_limit_test.py InferSizeLimitTest.test_json_dtype_size_expansion_exceeds_limit_error >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Default Input Size Limit Test Failed for type size explosion\n***" - RET=1 -fi - -# Test that sending multiple malformed compressed requests does not cause memory leaks on the server. -SERVER_PID=$SERVER_PID python http_input_size_limit_test.py InferSizeLimitTest.test_no_leak_on_invalid_inference_header_length >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Decompression Leak Regression Failed\n***\n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# Now run with increased size limit (128MB) - large inputs should succeed -SERVER_ARGS="--model-repository=${MODELDIR} --http-max-input-size=$((2**27))" -SERVER_LOG="./inference_server_increased_limit.log" -CLIENT_LOG="./http_input_size_limit_increased.log" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER with increased HTTP input size limit\n***" - cat $SERVER_LOG - exit 1 -fi - -rm -f $CLIENT_LOG -set +e -python http_input_size_limit_test.py InferSizeLimitTest.test_large_input_raw_binary >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Input Size Limit Test Failed for raw binary input with increased limits\n***" - RET=1 -fi - -python http_input_size_limit_test.py InferSizeLimitTest.test_large_input_json >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Input Size Limit Test Failed for JSON input with increased limits\n***" - RET=1 -fi - -python http_input_size_limit_test.py InferSizeLimitTest.test_large_input_compressed >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Input Size Limit Test Failed for compressed input with increased limits\n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# Test with zero max input size - should fail to start -SERVER_ARGS="--model-repository=${MODELDIR} --http-max-input-size=0" -SERVER_LOG="./inference_server_zero_limit.log" -CLIENT_LOG="./http_input_size_limit_zero.log" -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "\n***\n*** Server should not start with zero max input size\n***" - kill $SERVER_PID - wait $SERVER_PID - RET=1 -elif [ `grep -c "Error: --http-max-input-size must be greater than 0." ${SERVER_LOG}` != "1" ]; then - echo -e "\n***\n*** Failed. Expected '--http-max-input-size must be greater than 0' to be found in log\n***" - cat $SERVER_LOG - RET=1 -fi - -# Test with negative max input size - should fail to start -SERVER_ARGS="--model-repository=${MODELDIR} --http-max-input-size=-1024" -SERVER_LOG="./inference_server_negative_limit.log" -CLIENT_LOG="./http_input_size_limit_negative.log" -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "\n***\n*** Server should not start with negative max input size\n***" - kill $SERVER_PID - wait $SERVER_PID - RET=1 -elif [ `grep -c "Error: --http-max-input-size must be greater than 0." ${SERVER_LOG}` != "1" ]; then - echo -e "\n***\n*** Failed. Expected '--http-max-input-size must be greater than 0' to be found in log\n***" - cat $SERVER_LOG - RET=1 -fi - -### Test HTTP Requests Containing Many Chunks ### -MODELDIR="`pwd`/models" -REQUEST_MANY_CHUNKS_PY="http_request_many_chunks.py" -CLIENT_LOG="./client.http_request_many_chunks.log" -SERVER_ARGS="--model-repository=${MODELDIR} --allow-client-shm=true --log-verbose=1 --model-control-mode=explicit --load-model=simple" -SERVER_LOG="./inference_server_request_many_chunks.log" - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -SERVER_PID=$SERVER_PID python $REQUEST_MANY_CHUNKS_PY -v >> ${CLIENT_LOG} 2>&1 -if [ $? -ne 0 ]; then - echo -e "\n***\n*** HTTP Request Many Chunks Test Failed\n***" - cat $SERVER_LOG - cat $CLIENT_LOG - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID +### if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" diff --git a/qa/L0_http_fuzz/test.sh b/qa/L0_http_fuzz/test.sh index 96433aaed9..1abe08b487 100755 --- a/qa/L0_http_fuzz/test.sh +++ b/qa/L0_http_fuzz/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -65,7 +65,7 @@ function_install_python38() { # Install test script dependencies pip3 install --upgrade wheel setuptools boofuzz==0.3.0 "numpy<2" pillow attrdict future grpcio requests gsutil \ - awscli six grpcio-channelz prettytable virtualenv ml_dtypes + awscli six grpcio-channelz prettytable virtualenv } function_install_python38 diff --git a/qa/L0_infer/infer_test.py b/qa/L0_infer/infer_test.py index a195092c1b..8f16a7fe10 100755 --- a/qa/L0_infer/infer_test.py +++ b/qa/L0_infer/infer_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,7 +42,6 @@ TEST_CUDA_SHARED_MEMORY = bool(int(os.environ.get("TEST_CUDA_SHARED_MEMORY", 0))) CPU_ONLY = os.environ.get("TRITON_SERVER_CPU_ONLY") is not None TEST_VALGRIND = bool(int(os.environ.get("TEST_VALGRIND", 0))) -VALGRIND_TESTS = bool(int(os.environ.get("VALGRIND_TESTS", 0))) USE_GRPC = os.environ.get("USE_GRPC", 1) != "0" USE_HTTP = os.environ.get("USE_HTTP", 1) != "0" @@ -70,6 +69,7 @@ def _full_exact( output0_raw, output1_raw, swap, + network_timeout=NETWORK_TIMEOUT, ): def _infer_exact_helper( tester, @@ -157,13 +157,44 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size,), + (input_size,), + (input_size,), ): ensemble_prefix.append(prefix) + if tu.validate_for_onnx_model( + input_dtype, + output0_dtype, + output1_dtype, + (input_size,), + (input_size,), + (input_size,), + ): + for prefix in ensemble_prefix: + for pf in ["onnx"]: + if pf in BACKENDS: + _infer_exact_helper( + self, + prefix + pf, + (input_size,), + 8, + input_dtype, + output0_dtype, + output1_dtype, + output0_raw=output0_raw, + output1_raw=output1_raw, + swap=swap, + network_timeout=network_timeout, + ) + if not CPU_ONLY and tu.validate_for_trt_model( input_dtype, output0_dtype, output1_dtype, + (input_size, 1, 1), + (input_size, 1, 1), + (input_size, 1, 1), ): for prefix in ensemble_prefix: if "plan" in BACKENDS: @@ -198,6 +229,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size,), + (input_size,), + (input_size,), ): for prefix in ensemble_prefix: if "onnx" in BACKENDS: @@ -632,59 +666,16 @@ def test_mix_iff(self): swap=False, ) - if not VALGRIND_TESTS: - - def test_raw_version_latest_1(self): - input_size = 16 - tensor_shape = (1, input_size) - - # There are 3 versions of onnx_int8_int8_int8 but - # only version 3 should be available - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - try: - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int8, - np.int8, - np.int8, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) - - try: - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int8, - np.int8, - np.int8, - model_version=2, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) + def test_raw_version_latest_1(self): + input_size = 16 + tensor_shape = (1, input_size) + # There are 3 versions of onnx_int8_int8_int8 but + # only version 3 should be available + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + try: iu.infer_exact( self, platform, @@ -693,52 +684,25 @@ def test_raw_version_latest_1(self): np.int8, np.int8, np.int8, - model_version=3, - swap=True, + model_version=1, + swap=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) - def test_raw_version_latest_2(self): - input_size = 16 - tensor_shape = (1, input_size) - - # There are 3 versions of onnx_int16_int16_int16 but only - # versions 2 and 3 should be available - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - try: - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int16, - np.int16, - np.int16, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) - + try: iu.infer_exact( self, platform, tensor_shape, 1, - np.int16, - np.int16, - np.int16, + np.int8, + np.int8, + np.int8, model_version=2, swap=True, use_http=USE_HTTP, @@ -746,6 +710,35 @@ def test_raw_version_latest_2(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) + + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int8, + np.int8, + np.int8, + model_version=3, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + def test_raw_version_latest_2(self): + input_size = 16 + tensor_shape = (1, input_size) + + # There are 3 versions of onnx_int16_int16_int16 but only + # versions 2 and 3 should be available + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + try: iu.infer_exact( self, platform, @@ -754,31 +747,6 @@ def test_raw_version_latest_2(self): np.int16, np.int16, np.int16, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - def test_raw_version_all(self): - input_size = 16 - tensor_shape = (1, input_size) - - # There are 3 versions of onnx_int32_int32_int32 and all should - # be available. - for platform in ["onnx"]: - if platform not in BACKENDS: - continue - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.int32, - np.int32, - np.int32, model_version=1, swap=False, use_http=USE_HTTP, @@ -786,14 +754,129 @@ def test_raw_version_all(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) + + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int16, + np.int16, + np.int16, + model_version=2, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int16, + np.int16, + np.int16, + model_version=3, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + def test_raw_version_all(self): + input_size = 16 + tensor_shape = (1, input_size) + + # There are 3 versions of *_int32_int32_int32 and all should + # be available. + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int32, + np.int32, + np.int32, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int32, + np.int32, + np.int32, + model_version=2, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.int32, + np.int32, + np.int32, + model_version=3, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + def test_raw_version_specific_1(self): + input_size = 16 + tensor_shape = (1, input_size) + + # There are 3 versions of *_float16_float16_float16 but only + # version 1 should be available. + for platform in ["onnx"]: + if platform not in BACKENDS: + continue + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.float16, + np.float16, + np.float16, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + try: iu.infer_exact( self, platform, tensor_shape, 1, - np.int32, - np.int32, - np.int32, + np.float16, + np.float16, + np.float16, model_version=2, swap=True, use_http=USE_HTTP, @@ -801,14 +884,18 @@ def test_raw_version_all(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) + + try: iu.infer_exact( self, platform, tensor_shape, 1, - np.int32, - np.int32, - np.int32, + np.float16, + np.float16, + np.float16, model_version=3, swap=True, use_http=USE_HTTP, @@ -816,149 +903,143 @@ def test_raw_version_all(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) - def test_raw_version_specific_1(self): - input_size = 16 + def test_raw_version_specific_1_3(self): + input_size = 16 + + # There are 3 versions of *_float32_float32_float32 but only + # versions 1 and 3 should be available. + for platform in ("onnx", "plan"): + if platform == "plan" and CPU_ONLY: + continue + if platform not in BACKENDS: + continue tensor_shape = (1, input_size) + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.float32, + np.float32, + np.float32, + model_version=1, + swap=False, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) - # There are 3 versions of onnx_float16_float16_float16 but only - # version 1 should be available. - for platform in ["onnx"]: - if platform not in BACKENDS: - continue + try: iu.infer_exact( self, platform, tensor_shape, 1, - np.float16, - np.float16, - np.float16, - model_version=1, - swap=False, + np.float32, + np.float32, + np.float32, + model_version=2, + swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except InferenceServerException as ex: + self.assertTrue(ex.message().startswith("Request for unknown model")) + + iu.infer_exact( + self, + platform, + tensor_shape, + 1, + np.float32, + np.float32, + np.float32, + model_version=3, + swap=True, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) - try: + if ENSEMBLES: + if all(x in BACKENDS for x in ["onnx", "plan"]): + + def test_ensemble_mix_platform(self): + # Skip on CPU only machine as TensorRT model is used in this ensemble + if CPU_ONLY: + return + for bs in (1, 8): iu.infer_exact( self, - platform, - tensor_shape, - 1, - np.float16, - np.float16, - np.float16, - model_version=2, - swap=True, + "mix_platform", + (bs, 16), + bs, + np.float32, + np.float32, + np.float32, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) - try: + if "onnx" in BACKENDS: + + def test_ensemble_mix_type(self): + for bs in (1, 8): iu.infer_exact( self, - platform, - tensor_shape, - 1, - np.float16, - np.float16, - np.float16, - model_version=3, - swap=True, + "mix_type", + (bs, 16), + bs, + np.int32, + np.float32, + np.float32, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) - def test_raw_version_specific_1_3(self): - input_size = 16 - - # There are 3 versions of *_float32_float32_float32 but only - # versions 1 and 3 should be available. - for platform in ("onnx", "plan"): - if platform == "plan" and CPU_ONLY: - continue - if platform not in BACKENDS: - continue - tensor_shape = (1, input_size) - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.float32, - np.float32, - np.float32, - model_version=1, - swap=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) + if all(x in BACKENDS for x in ["onnx", "plan"]): - try: + def test_ensemble_mix_ensemble(self): + for bs in (1, 8): iu.infer_exact( self, - platform, - tensor_shape, - 1, - np.float32, + "mix_ensemble", + (bs, 16), + bs, + np.int32, np.float32, np.float32, - model_version=2, - swap=True, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - except InferenceServerException as ex: - self.assertTrue( - ex.message().startswith("Request for unknown model") - ) - - iu.infer_exact( - self, - platform, - tensor_shape, - 1, - np.float32, - np.float32, - np.float32, - model_version=3, - swap=True, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - if ENSEMBLES: - if all(x in BACKENDS for x in ["onnx", "libtorch"]): + if all( + x in BACKENDS + for x in [ + "onnx", + ] + ): - def test_ensemble_mix_platform(self): - # Skip on CPU only machine as TensorRT model is used in this ensemble - if CPU_ONLY: - return + def test_ensemble_mix_batch_nobatch(self): + base_names = ["batch_to_nobatch", "nobatch_to_batch"] + for name in base_names: for bs in (1, 8): iu.infer_exact( self, - "mix_platform", + name, (bs, 16), bs, np.float32, @@ -969,28 +1050,60 @@ def test_ensemble_mix_platform(self): use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + iu.infer_exact( + self, + name + "_nobatch", + (8, 16), + 1, + np.float32, + np.float32, + np.float32, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) + + # batch -> nobatch -> batch + for bs in (1, 8): + iu.infer_exact( + self, + "mix_nobatch_batch", + (bs, 16), + bs, + np.float32, + np.float32, + np.float32, + use_http=USE_HTTP, + use_grpc=USE_GRPC, + use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, + use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, + ) - if "onnx" in BACKENDS: + if not (TEST_SYSTEM_SHARED_MEMORY or TEST_CUDA_SHARED_MEMORY): - def test_ensemble_mix_type(self): + def test_ensemble_label_lookup(self): + if all(x in BACKENDS for x in ["onnx", "plan"]): + # Ensemble needs to look up label from the actual model for bs in (1, 8): iu.infer_exact( self, - "mix_type", + "mix_platform", (bs, 16), bs, - np.int32, np.float32, np.float32, + np.float32, + output0_raw=False, + output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if all(x in BACKENDS for x in ["onnx", "libtorch"]): - - def test_ensemble_mix_ensemble(self): + if all(x in BACKENDS for x in ["onnx", "plan"]): + # Label from the actual model will be passed along the nested ensemble for bs in (1, 8): iu.infer_exact( self, @@ -1000,148 +1113,55 @@ def test_ensemble_mix_ensemble(self): np.int32, np.float32, np.float32, + output0_raw=False, + output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if all( - x in BACKENDS - for x in [ - "onnx", - ] - ): - - def test_ensemble_mix_batch_nobatch(self): - base_names = ["batch_to_nobatch", "nobatch_to_batch"] - for name in base_names: - for bs in (1, 8): - iu.infer_exact( - self, - name, - (bs, 16), - bs, - np.float32, - np.float32, - np.float32, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) + if "onnx" in BACKENDS: + # If label file is provided, it will use the provided label file directly + try: iu.infer_exact( self, - name + "_nobatch", - (8, 16), + "wrong_label", + (1, 16), 1, + np.int32, np.float32, np.float32, - np.float32, + output0_raw=False, + output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) + except AssertionError: + # Sanity check that infer_exact failed since this ensemble is provided + # with unexpected labels + pass - # batch -> nobatch -> batch + if "onnx" in BACKENDS: for bs in (1, 8): iu.infer_exact( self, - "mix_nobatch_batch", + "label_override", (bs, 16), bs, + np.int32, np.float32, np.float32, - np.float32, + output0_raw=False, + output1_raw=False, use_http=USE_HTTP, use_grpc=USE_GRPC, use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if not (TEST_SYSTEM_SHARED_MEMORY or TEST_CUDA_SHARED_MEMORY): - - def test_ensemble_label_lookup(self): - if all(x in BACKENDS for x in ["onnx", "libtorch"]): - # Ensemble needs to look up label from the actual model - for bs in (1, 8): - iu.infer_exact( - self, - "mix_platform", - (bs, 16), - bs, - np.float32, - np.float32, - np.float32, - output0_raw=False, - output1_raw=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - if all(x in BACKENDS for x in ["onnx", "libtorch"]): - # Label from the actual model will be passed along the nested ensemble - for bs in (1, 8): - iu.infer_exact( - self, - "mix_ensemble", - (bs, 16), - bs, - np.int32, - np.float32, - np.float32, - output0_raw=False, - output1_raw=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - - if "onnx" in BACKENDS: - # If label file is provided, it will use the provided label file directly - try: - iu.infer_exact( - self, - "wrong_label", - (1, 16), - 1, - np.int32, - np.float32, - np.float32, - output0_raw=False, - output1_raw=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - except AssertionError: - # Sanity check that infer_exact failed since this ensemble is provided - # with unexpected labels - pass - - if "onnx" in BACKENDS: - for bs in (1, 8): - iu.infer_exact( - self, - "label_override", - (bs, 16), - bs, - np.int32, - np.float32, - np.float32, - output0_raw=False, - output1_raw=False, - use_http=USE_HTTP, - use_grpc=USE_GRPC, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_infer/test.sh b/qa/L0_infer/test.sh index 691eca4b72..79ba093ffd 100755 --- a/qa/L0_infer/test.sh +++ b/qa/L0_infer/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -73,7 +73,7 @@ fi if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="33"} else - EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="46"} + EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="44"} fi TEST_JETSON=${TEST_JETSON:=0} @@ -109,9 +109,6 @@ fi # Allow more time to exit. Ensemble brings in too many models SERVER_ARGS_EXTRA="--exit-timeout-secs=${SERVER_TIMEOUT} --backend-directory=${BACKEND_DIR} --backend-config=python,stub-timeout-seconds=120 --backend-config=python,shm-default-byte-size=${DEFAULT_SHM_SIZE_BYTES}" -if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then - SERVER_ARGS_EXTRA="${SERVER_ARGS_EXTRA} --allow-client-shm=true" -fi SERVER_ARGS="--model-repository=${MODELDIR} ${SERVER_ARGS_EXTRA}" SERVER_LOG_BASE="./inference_server" source ../common/util.sh @@ -163,9 +160,6 @@ function generate_model_repository() { # Types that need to use SubAdd instead of AddSub swap_types="float32 int32 int16 int8" for onnx_model in $onnx_models; do - # Skip BF16 models: Not yet supported by Python backend - [[ "$onnx_model" == *bf16* ]] && continue - if [ "$BACKEND" == "python_dlpack" ]; then python_model=`echo $onnx_model | sed 's/onnx/python_dlpack/g' | sed 's,'"$DATADIR/qa_model_repository/"',,g'` else @@ -218,10 +212,6 @@ function generate_model_repository() { else cp -r ${DATADIR}/qa_model_repository/${BACKEND}* \ models/. - # Remove ONNX BF16 models from CPU models - if [ "$BACKEND" == "onnx" ] && [ "$TARGET" == "cpu" ]; then - rm -rf models/onnx_*bf16* - fi fi done @@ -260,24 +250,20 @@ function generate_model_repository() { KIND="KIND_GPU" && [[ "$TARGET" == "cpu" ]] && KIND="KIND_CPU" for FW in $BACKENDS; do - [ "$FW" == "plan" ] && continue - for MC in `ls models/${FW}*/config.pbtxt`; do - # BF16 models: ORT CPU has no BF16 kernels, force GPU - if [ "$FW" == "onnx" ] && [ "$MC" == *bf16* ]; then - MC_KIND="KIND_GPU" - else - MC_KIND=${KIND} - fi - - if [ "$FW" == "onnx" ] && [ "$TEST_VALGRIND" -eq 1 ]; then - # Reduce the instance count to make loading onnx models faster - echo "instance_group [ { kind: ${MC_KIND} count: 1 }]" >> $MC - elif [ "$FW" == "python" ] || [ "$FW" == "python_dlpack" ] || [ "$FW" == "openvino" ]; then - echo "instance_group [ { kind: KIND_CPU }]" >> $MC - else - echo "instance_group [ { kind: ${KIND} }]" >> $MC - fi - done + if [ "$FW" == "onnx" ] && [ "$TEST_VALGRIND" -eq 1 ]; then + # Reduce the instance count to make loading onnx models faster + for MC in `ls models/${FW}*/config.pbtxt`; do + echo "instance_group [ { kind: ${KIND} count: 1 }]" >> $MC + done + elif [ "$FW" != "plan" ] && [ "$FW" != "python" ] && [ "$FW" != "python_dlpack" ] && [ "$FW" != "openvino" ];then + for MC in `ls models/${FW}*/config.pbtxt`; do + echo "instance_group [ { kind: ${KIND} }]" >> $MC + done + elif [ "$FW" == "python" ] || [ "$FW" == "python_dlpack" ] || [ "$FW" == "openvino" ]; then + for MC in `ls models/${FW}*/config.pbtxt`; do + echo "instance_group [ { kind: KIND_CPU }]" >> $MC + done + fi done # Modify custom_zero_1_float32 and custom_nobatch_zero_1_float32 for relevant ensembles @@ -363,7 +349,7 @@ done # separately to reduce the loading time. if [ "$TEST_VALGRIND" -eq 1 ]; then TESTING_BACKENDS="python python_dlpack onnx" - EXPECTED_NUM_TESTS=36 + EXPECTED_NUM_TESTS=42 if [[ "aarch64" != $(uname -m) ]] ; then pip3 install torch==2.3.1+cpu -f https://download.pytorch.org/whl/torch_stable.html else @@ -378,11 +364,6 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then mkdir nobatch_models mv ./models/*nobatch_* ./nobatch_models/. cp -fr ./models/nop_* ./nobatch_models/. - if [[ $BACKENDS == *"onnx"* ]]; then - # These two models are required by test_ensemble_mix_batch_nobatch test case. - cp -fr ./models/onnx_float32_float32_float32 ./nobatch_models/. - cp -fr ./models/custom_zero_1_float32 ./nobatch_models/. - fi for BATCHING_MODE in batch nobatch; do if [ "$TRITON_SERVER_CPU_ONLY" == "1" ]; then @@ -422,7 +403,7 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then set +e - VALGRIND_TESTS="1" python3 $INFER_TEST >$CLIENT_LOG 2>&1 + python3 $INFER_TEST >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG RET=1 diff --git a/qa/L0_infer_reshape/infer_reshape_test.py b/qa/L0_infer_reshape/infer_reshape_test.py index d3adcfbbeb..5445f5d6c9 100755 --- a/qa/L0_infer_reshape/infer_reshape_test.py +++ b/qa/L0_infer_reshape/infer_reshape_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,7 +50,9 @@ def _full_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True): output_shapes = input_shapes # For validation assume any shape can be used... - if tu.validate_for_onnx_model(dtype, dtype, dtype): + if tu.validate_for_onnx_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): # model that supports batching for bs in (1, 8): full_shapes = [ @@ -153,6 +155,9 @@ def _full_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True): dtype, dtype, dtype, + input_shapes[0], + input_shapes[0], + input_shapes[0], ): # model that supports batching for bs in (1, 8): @@ -198,7 +203,9 @@ def _trt_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True): if output_shapes is None: output_shapes = input_shapes - if tu.validate_for_trt_model(dtype, dtype, dtype): + if tu.validate_for_trt_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): # model that supports batching for bs in (1, 8): full_shapes = [ diff --git a/qa/L0_infer_variable/infer_variable_test.py b/qa/L0_infer_variable/infer_variable_test.py index a3dce782df..55a2c7a084 100755 --- a/qa/L0_infer_variable/infer_variable_test.py +++ b/qa/L0_infer_variable/infer_variable_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -140,6 +140,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + input_shape, + input_shape, + input_shape, ): ensemble_prefix.append(prefix) @@ -147,6 +150,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): for prefix in ensemble_prefix: if input_dtype == np.int8: @@ -180,6 +186,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): # No basic ensemble models are created against custom models [TODO] _infer_exact_helper( diff --git a/qa/L0_infer_variable/test.sh b/qa/L0_infer_variable/test.sh index 4498f1a57f..36d86716ad 100755 --- a/qa/L0_infer_variable/test.sh +++ b/qa/L0_infer_variable/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -64,11 +64,6 @@ for TARGET in cpu gpu; do cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository models && \ cp -r /data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_variable_model_repository/* models/. - # Remove ONNX BF16 models from CPU models - if [ "$TARGET" == "cpu" ]; then - rm -rf models/onnx_*bf16* - fi - create_nop_version_dir `pwd`/models KIND="KIND_GPU" && [[ "$TARGET" == "cpu" ]] && KIND="KIND_CPU" diff --git a/qa/L0_infer_zero/infer_zero_test.py b/qa/L0_infer_zero/infer_zero_test.py index 89c77f2554..d05b383cff 100755 --- a/qa/L0_infer_zero/infer_zero_test.py +++ b/qa/L0_infer_zero/infer_zero_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -93,7 +93,9 @@ def _full_zero(self, dtype, shapes): ) for name in ["simple_zero", "sequence_zero", "fan_zero"]: - if tu.validate_for_ensemble_model(name, dtype, dtype, dtype): + if tu.validate_for_ensemble_model( + name, dtype, dtype, dtype, shapes[0], shapes[0], shapes[0] + ): # model that supports batching for bs in (1, 8): batch_shapes = [ diff --git a/qa/L0_input_validation/input_validation_test.py b/qa/L0_input_validation/input_validation_test.py index 0ac4dbe1b8..e3524247aa 100755 --- a/qa/L0_input_validation/input_validation_test.py +++ b/qa/L0_input_validation/input_validation_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -288,90 +288,5 @@ def inference_helper(model_name, batch_size=1): inference_helper(model_name="plan_zero_1_float32_int32", batch_size=8) -class ModelNameValidationTest(unittest.TestCase): - INVALID_TRAVERSAL_NAMES = [ - "../etc", - "a/../b", - "../../etc/passwd", - "../../../../etc", - "model/..", - "..", - "/etc/passwd", - "model/subdir", - "model/", - " ..", - ".. ", - ] - - def test_model_name_invalid_load(self): - client = tritongrpcclient.InferenceServerClient("localhost:8001") - for model_name in self.INVALID_TRAVERSAL_NAMES: - print(f"Testing model name: {model_name!r}") - with self.assertRaises(InferenceServerException) as cm: - client.load_model(model_name) - self.assertIn( - "model name must not contain path traversal characters", - str(cm.exception), - f"Expected traversal rejection for model name: {model_name!r}", - ) - - def test_model_name_empty_load(self): - client = tritongrpcclient.InferenceServerClient("localhost:8001") - with self.assertRaises(InferenceServerException) as cm: - client.load_model("") - self.assertIn( - "Model name cannot be empty. Please enter a valid name to deploy.", - str(cm.exception), - ) - - def test_model_name_whitespace_only_load(self): - client = tritongrpcclient.InferenceServerClient("localhost:8001") - whitespace_names = [" ", " ", "\t", "\n", "\r", "\f", "\v", " \t \n "] - for model_name in whitespace_names: - with self.assertRaises(InferenceServerException) as cm: - client.load_model(model_name) - self.assertIn( - "Model name cannot be empty. Please enter a valid name to deploy.", - str(cm.exception), - f"Expected whitespace-only rejection for model name: {model_name!r}", - ) - - def test_model_name_invalid_unload(self): - # Unload should not trigger traversal check - client = tritongrpcclient.InferenceServerClient("localhost:8001") - for model_name in self.INVALID_TRAVERSAL_NAMES: - try: - client.unload_model(model_name) - except InferenceServerException as e: - self.assertNotIn( - "model name must not contain path traversal characters", - str(e), - f"Unload should not trigger traversal rejection for model name: {model_name!r}", - ) - - def test_model_name_valid(self): - """Verify that a syntactically valid model name is not rejected by - the traversal check -- it should fail with a model not found error instead.""" - VALID_MODEL_NAMES = [ - "model123", - # "model OAI", TRI-769: Fix this test case - "model.version1", - "...", - "..my_model", - "model..1", - "model....1", - ] - client = tritongrpcclient.InferenceServerClient("localhost:8001") - for model_name in VALID_MODEL_NAMES: - with self.assertRaises(InferenceServerException) as cm: - client.load_model(model_name) - self.assertNotIn( - "path traversal characters", - str(cm.exception), - "Valid model name should not trigger path traversal rejection", - ) - self.assertIn("failed to poll from model repository", str(cm.exception)) - - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_input_validation/test.sh b/qa/L0_input_validation/test.sh index 767d49988d..2d2cf515ce 100755 --- a/qa/L0_input_validation/test.sh +++ b/qa/L0_input_validation/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,6 +48,8 @@ CLIENT_LOG="./input_validation_client.log" TEST_PY=./input_validation_test.py TEST_RESULT_FILE='./test_results.txt' SERVER_LOG="./inference_server.log" +TEST_LOG="./input_byte_size_test.log" +TEST_EXEC=./input_byte_size_test export CUDA_VISIBLE_DEVICES=0 @@ -126,7 +128,7 @@ cp -r $DATADIR/qa_model_repository/onnx_object_int32_int32 models/. cp -r $DATADIR/qa_shapetensor_model_repository/plan_nobatch_zero_1_float32_int32 models/. cp -r $DATADIR/qa_shapetensor_model_repository/plan_zero_1_float32_int32 models/. -SERVER_ARGS="--model-repository=`pwd`/models --allow-client-shm=true" +SERVER_ARGS="--model-repository=`pwd`/models" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -149,9 +151,7 @@ kill $SERVER_PID wait $SERVER_PID # input_byte_size_test -TEST_LOG="./input_byte_size_test.log" -TEST_EXEC=./input_byte_size_test -cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/{onnx_zero_1_float32,onnx_zero_1_object,onnx_zero_1_bool} ./models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/{onnx_zero_1_float32,onnx_zero_1_object} ./models set +e LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH $TEST_EXEC >> $TEST_LOG 2>&1 @@ -162,45 +162,6 @@ if [ $? -ne 0 ]; then fi set -e -# tensor_size_test -TEST_LOG="./tensor_size_test.log" -TEST_EXEC=./tensor_size_test - -set +e -LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH $TEST_EXEC >> $TEST_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $TEST_LOG - echo -e "\n***\n*** tensor_size_test FAILED\n***" - RET=1 -fi -set -e - -# Model name validation test -rm -rf test_models ; mkdir -p test_models -SERVER_LOG="./model_name_validation_server.log" -CLIENT_LOG="./model_name_validation_client.log" -SERVER_ARGS="--model-repository=`pwd`/test_models --model-control-mode=explicit --log-verbose=1" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python3 -m pytest -s --junitxml="model_name_validation.report.xml" $TEST_PY::ModelNameValidationTest >> $CLIENT_LOG 2>&1 - -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - cat $SERVER_LOG - echo -e "\n***\n*** input_validation_test.py::ModelNameValidationTest FAILED. \n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Input Validation Test Passed\n***" else diff --git a/qa/L0_io/test.sh b/qa/L0_io/test.sh index 43c7e7dd41..10318da4db 100755 --- a/qa/L0_io/test.sh +++ b/qa/L0_io/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,7 +55,7 @@ LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH rm -f $CLIENT_LOG* # PyTorch is required for the Python backend dlpack add sub models -pip3 install torch -f https://download.pytorch.org/whl/cu130 +pip3 install torch==2.3.1+cu118 -f https://download.pytorch.org/whl/torch_stable.html RET=0 # Prepare float32 models with basic config diff --git a/qa/L0_libtorch_instance_group_kind_model/test.sh b/qa/L0_libtorch_instance_group_kind_model/test.sh index a68a8b3257..9cbb6bf550 100755 --- a/qa/L0_libtorch_instance_group_kind_model/test.sh +++ b/qa/L0_libtorch_instance_group_kind_model/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -39,7 +39,7 @@ if [ ! -z "$TEST_REPO_ARCH" ]; then fi pip3 uninstall -y torch -pip3 install torch -f https://download.pytorch.org/whl/cu130 +pip3 install torch==2.3.1+cu118 -f https://download.pytorch.org/whl/torch_stable.html DATADIR=/data/inferenceserver/${REPO_VERSION}/qa_model_repository SERVER=/opt/tritonserver/bin/tritonserver diff --git a/qa/L0_lifecycle/lifecycle_test.py b/qa/L0_lifecycle/lifecycle_test.py index 2dba1c25d5..8ad95e3d90 100755 --- a/qa/L0_lifecycle/lifecycle_test.py +++ b/qa/L0_lifecycle/lifecycle_test.py @@ -2669,8 +2669,7 @@ def callback(user_data, result, error): self.assertTrue(False, "expected error for new inference during shutdown") except InferenceServerException as ex: self.assertIn( - "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + "Server is stopping, scheduler for model has stopped accepting new inference requests", ex.message(), ) @@ -2732,29 +2731,33 @@ def callback(user_data, result, error): ) self.assertTrue(False, "expected error for new inference during shutdown") except InferenceServerException as ex: - # The first request received by the gRPC endpoint while shutting down returns CANCELLED - # each subsequent request returns Connection refused - self.assertIn("CANCELLED", ex.message()) + self.assertIn( + "Server is stopping, scheduler for model has stopped accepting new inference requests", + ex.message(), + ) # 2: New sequence with existing sequence ID try: triton_client.infer(model_name, inputs, sequence_id=1, sequence_start=True) self.assertTrue(False, "expected error for new inference during shutdown") except InferenceServerException as ex: self.assertIn( - "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + "Server is stopping, scheduler for model has stopped accepting new inference requests", ex.message(), ) - # 3: Continuing sequence after shutdown + # 3: Continuing sequence try: - triton_client.infer(model_name, inputs, sequence_id=2, sequence_end=True) - self.assertTrue(False, "expected error for new inference during shutdown") - except InferenceServerException as ex: - self.assertIn( - "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", - ex.message(), + res = triton_client.infer( + model_name, inputs, sequence_id=2, sequence_end=True ) + output_data = res.as_numpy("OUTPUT") + # Result are accumulated + np.testing.assert_allclose( + output_data, + input_data + input_data, + err_msg="Inference result is not correct", + ) + except Exception as ex: + self.assertTrue(False, "unexpected error {}".format(ex)) # Wait until the results are available in user_data time_out = 30 @@ -2813,9 +2816,9 @@ def callback(user_data, result, error): triton_client.infer(model_name, inputs) self.assertTrue(False, "expected error for new inference during shutdown") except InferenceServerException as ex: + self.assertIn("in ensemble 'ensemble_zero_1_float32'", ex.message()) self.assertIn( - "failed to connect to all addresses; last error: UNKNOWN: ipv4:127.0.0.1:8001: " - + "Failed to connect to remote host: connect: Connection refused (111)", + "Server is stopping, scheduler for model has stopped accepting new inference requests", ex.message(), ) @@ -3385,7 +3388,7 @@ def test_shutdown_with_live_connection(self): # close connection conn.close() - time.sleep(3) + time.sleep(2) # check exit timeout countdown did not restart with open(os.environ["SERVER_LOG"]) as f: diff --git a/qa/L0_lifecycle/test.sh b/qa/L0_lifecycle/test.sh index 964986a519..bcd1713bb8 100755 --- a/qa/L0_lifecycle/test.sh +++ b/qa/L0_lifecycle/test.sh @@ -1579,8 +1579,8 @@ check_unit_test set -e # check server log -if [ `grep -c "Found 1 gRPC service connections and inference handlers" $SERVER_LOG` == "0" ]; then - echo -e "\n***\n*** Expect logging for in-flight gRPC connection count\n***" +if [ `grep -c "Model 'custom_zero_1_float32' (version 1) has 1 in-flight inferences" $SERVER_LOG` == "0" ]; then + echo -e "\n***\n*** Expect logging for model and in-flight inference count\n***" RET=1 fi @@ -1617,6 +1617,10 @@ if [ `grep -c "Model 'custom_sequence_int32' (version 1) has 2 in-flight inferen echo -e "\n***\n*** Expect logging for model having 2 in-flight inferences\n***" RET=1 fi +if [ `grep -c "Model 'custom_sequence_int32' (version 1) has 1 in-flight inferences" $SERVER_LOG` == "0" ]; then + echo -e "\n***\n*** Expect logging for model having 1 in-flight inference\n***" + RET=1 +fi kill $SERVER_PID || true wait $SERVER_PID @@ -1654,8 +1658,8 @@ check_unit_test set -e # check server log -if [ `grep -c "Found 1 gRPC service connections and inference handlers" $SERVER_LOG` == "0" ]; then - echo -e "\n***\n*** Expect logging for in-flight gRPC connection count\n***" +if [ `grep -c "Model 'ensemble_zero_1_float32' (version 1) has 1 in-flight inferences" $SERVER_LOG` == "0" ]; then + echo -e "\n***\n*** Expect logging for model and in-flight inference count\n***" RET=1 fi @@ -1666,7 +1670,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_load_gpu_limit # dependency of the Python model to be used -pip install "cuda-python>=12,<13" +pip install cuda-python rm -fr models config.pbtxt.* mkdir models cp -r ../python_models/cuda_memory_consumer models/cuda_memory_consumer_1 && \ diff --git a/qa/L0_logging/test.sh b/qa/L0_logging/test.sh index 4dabe976e6..a25693cf0e 100755 --- a/qa/L0_logging/test.sh +++ b/qa/L0_logging/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -490,7 +490,7 @@ set -e kill $SERVER_PID wait $SERVER_PID -# Test Negative Test Cases +#Test Negative Test Cases SERVER_ARGS="--log-warn="false" --model-repository=$MODELSDIR" SERVER_LOG="./server.log" run_server @@ -607,53 +607,6 @@ fi set -e -# Test Log Output Stream -# Set up an invalid model with a leading zero in the version number. This will print warning and error logs. -MODELSDIR_INVALID=`pwd`/log_models_invalid -rm -rf $MODELSDIR_INVALID && \ - cp -r $MODELSDIR $MODELSDIR_INVALID && \ - mv $MODELSDIR_INVALID/simple/1 $MODELSDIR_INVALID/simple/01 - -rm -f log_file.log -LOG_REGEX="(?P\d{2})(?P\d{2}) (?P\d{2}:\d{2}:\d{2}\.\d{6}) (?P\d+) (?P[\w\.]+):(?P\d+)] (?P.*)" -SERVER_ARGS="--log-verbose=1 --model-repository=$MODELSDIR_INVALID" -SERVER_LOG="./inference_server_log_file.log" -SERVER_ERROR_LOG="./inference_server_error_log_file.log" -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - cat $SERVER_LOG - kill_server - exit 1 -fi - -set +e -# Only INFO logs in SERVER_LOG -if [ `grep -c -P "I$LOG_REGEX" $SERVER_LOG` == "0" ]; then - echo -e "\n***\n*** Test Failed: INFO logs are not written to $SERVER_LOG\n***" - RET=1 -fi -if [ `grep -c -P "(W|E)$LOG_REGEX" $SERVER_LOG` != "0" ]; then - echo -e "\n***\n*** Test Failed: WARNING/ERROR logs are written to $SERVER_LOG\n***" - RET=1 -fi -# Only WARNING and ERROR logs in SERVER_ERROR_LOG -if [ `grep -c -P "I$LOG_REGEX" $SERVER_ERROR_LOG` != "0" ]; then - echo -e "\n***\n*** Test Failed: INFO logs are written to $SERVER_ERROR_LOG\n***" - RET=1 -fi -if [ `grep -c -P "W$LOG_REGEX" $SERVER_ERROR_LOG` == "0" ]; then - echo -e "\n***\n*** Test Failed: ERROR logs are not written to $SERVER_ERROR_LOG\n***" - RET=1 -fi -if [ `grep -c -P "E$LOG_REGEX" $SERVER_ERROR_LOG` == "0" ]; then - echo -e "\n***\n*** Test Failed: ERROR logs are not written to $SERVER_ERROR_LOG\n***" - RET=1 -fi - -unset $SERVER_ERROR_LOG -set -e - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else diff --git a/qa/L0_long_running_stress/scenarios.py b/qa/L0_long_running_stress/scenarios.py index 342b8a6937..650f228ab5 100755 --- a/qa/L0_long_running_stress/scenarios.py +++ b/qa/L0_long_running_stress/scenarios.py @@ -99,7 +99,7 @@ def run(self, client_metadata): class PerfAnalyzerScenario(Scenario): # Some class static variables - command_ = "perf_analyzer" + command_ = "../clients/perf_analyzer" generation_mutex_ = threading.Lock() class ModelOption: @@ -203,7 +203,7 @@ def __init__( # Add no validation models self.options_.append( PerfAnalyzerScenario.ModelOption( - "resnet_v1_50", 32, (1, 4, 1), queue_latency_range_us + "resnet_v1_50_def", 32, (1, 4, 1), queue_latency_range_us ) ) for trial in sequence_trials: @@ -334,7 +334,7 @@ def run(self, client_metadata): class ResNetScenario(Scenario): def __init__(self, name, batch_size=32, verbose=False, out_stream=sys.stdout): super().__init__(name, [], verbose, out_stream) - self.model_name_ = "resnet_v1_50" + self.model_name_ = "resnet_v1_50_def" self.batch_size_ = batch_size img = self.preprocess("../images/vulture.jpeg") @@ -353,7 +353,7 @@ def preprocess(self, filename): return scaled def postprocess(self, results): - output_array = results.as_numpy("resnet_v1_50/predictions/Softmax:0") + output_array = results.as_numpy("resnet_v1_50/predictions/Softmax") if len(output_array) != self.batch_size_: raise Exception( "expected {} results, got {}".format( @@ -377,12 +377,12 @@ def postprocess(self, results): def run(self, client_metadata): triton_client = client_metadata[0] - inputs = [grpcclient.InferInput("input:0", self.image_data_.shape, "FP32")] + inputs = [grpcclient.InferInput("input", self.image_data_.shape, "FP32")] inputs[0].set_data_from_numpy(self.image_data_) outputs = [ grpcclient.InferRequestedOutput( - "resnet_v1_50/predictions/Softmax:0", class_count=1 + "resnet_v1_50/predictions/Softmax", class_count=1 ) ] res = triton_client.infer(self.model_name_, inputs, outputs=outputs) diff --git a/qa/L0_long_running_stress/test.sh b/qa/L0_long_running_stress/test.sh index 92d8c09b0d..1973028bcf 100755 --- a/qa/L0_long_running_stress/test.sh +++ b/qa/L0_long_running_stress/test.sh @@ -130,8 +130,9 @@ cp -r ../custom_models/custom_zero_1_float32 $MODEL_DIR/custom_zero_1_float32 && echo "{ key: \"execute_delay_ms\"; value: { string_value: \"10000\" }}" >> config.pbtxt && \ echo "]" >> config.pbtxt) -cp -r $DATADIR/onnx_model_store/resnet_v1_50 $MODEL_DIR/. && \ - (cd $MODEL_DIR/resnet_v1_50 && \ +cp -r $DATADIR/onnx_model_store/resnet_v1_50 $MODEL_DIR/resnet_v1_50_def && \ + (cd $MODEL_DIR/resnet_v1_50_def && \ + sed -i 's/^name: "resnet_v1_50"/name: "resnet_v1_50_def"/' config.pbtxt && \ echo "optimization { }" >> config.pbtxt) SERVER_ARGS="--model-repository=`pwd`/$MODEL_DIR" diff --git a/qa/L0_memory_growth/test.sh b/qa/L0_memory_growth/test.sh index f23813937d..fe4911c9bc 100755 --- a/qa/L0_memory_growth/test.sh +++ b/qa/L0_memory_growth/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,8 +42,7 @@ fi export CUDA_VISIBLE_DEVICES=0 # Clients -pip3 install perf_analyzer -PERF_ANALYZER=perf_analyzer +PERF_ANALYZER=../clients/perf_analyzer IMAGE=../images/vulture.jpeg # Models @@ -102,7 +101,7 @@ export MAX_ALLOWED_ALLOC="100" # Create local model repository mkdir -p models/ -cp -r $DATADIR/perf_model_store/resnet50_* models/ +cp -r $DATADIR/perf_model_store/resnet50* models/ # Create the TensorRT plan from ONNX model rm -fr models/resnet50_fp32_plan && mkdir -p models/resnet50_fp32_plan/1 && \ @@ -132,9 +131,7 @@ RET=0 for MODEL in $(ls models); do # Skip the resnet50_fp32_libtorch model as it is running into `misaligned address' # Tracked here: https://nvbugs/3954104 - # Skip the resnet50_fp32_onnx model as the inference hangs on A100 with batch size > 1. - # Tracked here: https://linear.app/nvidia/issue/TRI-304 - if [[ "$MODEL" == "resnet50_fp32_libtorch" || "$MODEL" == "resnet50_fp32_onnx" ]]; then + if [ "$MODEL" == "resnet50_fp32_libtorch" ]; then continue fi diff --git a/qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py b/qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py index 73454c5a3c..6a73e12da4 100644 --- a/qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py +++ b/qa/L0_metrics/ensemble_decoupled/async_execute_decouple/1/model.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,9 +45,9 @@ async def execute(self, requests): ] # Wait + time.sleep(wait_secs.item()) response_sender = request.get_response_sender() for i in range(response_num): - time.sleep(wait_secs.item()) response = pb_utils.InferenceResponse(output_tensors) if i != response_num - 1: response_sender.send(response) diff --git a/qa/L0_metrics/histogram_metrics_test.py b/qa/L0_metrics/histogram_metrics_test.py index 725f86fcbe..a59aac0478 100755 --- a/qa/L0_metrics/histogram_metrics_test.py +++ b/qa/L0_metrics/histogram_metrics_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -114,7 +114,7 @@ def callback(user_data, result, error): def test_ensemble_decoupled(self): wait_secs = 1 responses_per_req = 3 - total_iters = 3 + total_reqs = 3 delta = 0.2 # Infer @@ -133,7 +133,7 @@ def test_ensemble_decoupled(self): inputs[1].set_data_from_numpy(input_data_1) # Send requests to ensemble decoupled model - for iter_cnt in range(1, total_iters + 1): + for request_num in range(1, total_reqs + 1): ensemble_model_name = "ensemble" decoupled_model_name = "async_execute_decouple" non_decoupled_model_name = "async_execute" @@ -144,40 +144,40 @@ def test_ensemble_decoupled(self): # Checks metrics output histogram_dict = self.get_histogram_metrics(FIRST_RESPONSE_HISTOGRAM) - def check_histogram(model_name, request_cnt, wait_secs_per_req, delta): - histogram_count_key = get_histogram_metric_key( + def check_existing_metrics(model_name, wait_secs_per_req, delta): + metric_count = get_histogram_metric_key( FIRST_RESPONSE_HISTOGRAM, model_name, "1", "count" ) - histogram_sum_key = get_histogram_metric_key( + metric_sum = get_histogram_metric_key( FIRST_RESPONSE_HISTOGRAM, model_name, "1", "sum" ) # Test histogram count - self.assertIn(histogram_count_key, histogram_dict) - self.assertEqual( - histogram_dict[histogram_count_key], request_cnt * iter_cnt - ) + self.assertIn(metric_count, histogram_dict) + self.assertEqual(histogram_dict[metric_count], request_num) # Test histogram sum - self.assertIn(histogram_sum_key, histogram_dict) + self.assertIn(metric_sum, histogram_dict) self.assertTrue( - wait_secs_per_req * MILLIS_PER_SEC * request_cnt * iter_cnt - <= histogram_dict[histogram_sum_key] - < (wait_secs_per_req + delta) - * MILLIS_PER_SEC - * request_cnt - * iter_cnt + wait_secs_per_req * MILLIS_PER_SEC * request_num + <= histogram_dict[metric_sum] + < (wait_secs_per_req + delta) * MILLIS_PER_SEC * request_num ) # Prometheus histogram buckets are tested in metrics_api_test.cc::HistogramAPIHelper # Test ensemble model metrics - check_histogram(ensemble_model_name, 1, wait_secs * 2, 2 * delta) + check_existing_metrics(ensemble_model_name, 2 * wait_secs, 2 * delta) # Test decoupled model metrics - check_histogram(decoupled_model_name, 1, wait_secs, delta) + check_existing_metrics(decoupled_model_name, wait_secs, delta) # Test non-decoupled model metrics - check_histogram( - non_decoupled_model_name, responses_per_req, wait_secs, delta + non_decoupled_model_count = get_histogram_metric_key( + FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "count" + ) + non_decoupled_model_sum = get_histogram_metric_key( + FIRST_RESPONSE_HISTOGRAM, non_decoupled_model_name, "1", "sum" ) + self.assertNotIn(non_decoupled_model_count, histogram_dict) + self.assertNotIn(non_decoupled_model_sum, histogram_dict) def test_buckets_override(self): model_name = "async_execute_decouple" diff --git a/qa/L0_metrics/metrics_config_test.py b/qa/L0_metrics/metrics_config_test.py index 0c448dbaa7..975d219ef8 100755 --- a/qa/L0_metrics/metrics_config_test.py +++ b/qa/L0_metrics/metrics_config_test.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -44,7 +44,7 @@ "nv_inference_compute_infer_duration", "nv_inference_compute_output_duration", ] -INF_HISTOGRAM_PATTERNS = ["nv_inference_first_response_histogram_ms"] +INF_HISTOGRAM_DECOUPLED_PATTERNS = ["nv_inference_first_response_histogram_ms"] INF_SUMMARY_PATTERNS = [ "nv_inference_request_summary", "nv_inference_queue_summary", @@ -99,15 +99,15 @@ def test_cache_counters_missing(self): self.assertNotIn(metric, metrics) # Histograms - def test_inf_histograms_exist(self): + def test_inf_histograms_decoupled_exist(self): metrics = self._get_metrics() - for metric in INF_HISTOGRAM_PATTERNS: + for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS: for suffix in ["_count", "_sum", "_bucket"]: self.assertIn(metric + suffix, metrics) - def test_inf_histograms_missing(self): + def test_inf_histograms_decoupled_missing(self): metrics = self._get_metrics() - for metric in INF_HISTOGRAM_PATTERNS: + for metric in INF_HISTOGRAM_DECOUPLED_PATTERNS: self.assertNotIn(metric, metrics) # Summaries diff --git a/qa/L0_metrics/test.sh b/qa/L0_metrics/test.sh index b1f77774a2..2d6e85e211 100755 --- a/qa/L0_metrics/test.sh +++ b/qa/L0_metrics/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -276,7 +276,7 @@ SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off" run_and_check_server python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG} check_unit_test -python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG} +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG} check_unit_test python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG} check_unit_test @@ -286,12 +286,47 @@ python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee check_unit_test kill_server -# Enable histograms +# Check default settings: Histograms should be always disabled in non-decoupled model. SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=identity_cache_off --metrics-config histogram_latencies=true" run_and_check_server python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG} check_unit_test -python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_exist 2>&1 | tee ${CLIENT_LOG} +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +kill_server + +# Check default settings: Histograms should be disabled in decoupled model +decoupled_model="async_execute_decouple" +mkdir -p "${MODELDIR}/${decoupled_model}/1/" +cp ../python_models/${decoupled_model}/model.py ${MODELDIR}/${decoupled_model}/1/ +cp ../python_models/${decoupled_model}/config.pbtxt ${MODELDIR}/${decoupled_model}/ + +SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model}" +run_and_check_server +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_counters_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_cache_summaries_missing 2>&1 | tee ${CLIENT_LOG} +check_unit_test +kill_server + +# Enable histograms in decoupled model +SERVER_ARGS="${BASE_SERVER_ARGS} --load-model=${decoupled_model} --metrics-config histogram_latencies=true" +run_and_check_server +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_counters_exist 2>&1 | tee ${CLIENT_LOG} +check_unit_test +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_exist 2>&1 | tee ${CLIENT_LOG} check_unit_test python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_summaries_missing 2>&1 | tee ${CLIENT_LOG} check_unit_test @@ -430,7 +465,6 @@ SERVER_LOG="./histogram_ensemble_decoupled_server.log" CLIENT_LOG="./histogram_ensemble_decoupled_client.log" SERVER_ARGS="--model-repository=${MODELDIR} --metrics-config histogram_latencies=true --log-verbose=1" mkdir -p "${MODELDIR}"/ensemble/1 -rm -rf "${MODELDIR}"/async_execute cp -r "${MODELDIR}"/async_execute_decouple "${MODELDIR}"/async_execute sed -i "s/model_transaction_policy { decoupled: True }//" "${MODELDIR}"/async_execute/config.pbtxt @@ -476,7 +510,7 @@ kill_server PYTHON_TEST="metrics_config_test.py" SERVER_ARGS="--model-repository=${MODELDIR} --model-control-mode=explicit --load-model=${decoupled_model} --metrics-config histogram_latencies=false --log-verbose=1" run_and_check_server -python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_missing 2>&1 | tee ${CLIENT_LOG} +python3 ${PYTHON_TEST} MetricsConfigTest.test_inf_histograms_decoupled_missing 2>&1 | tee ${CLIENT_LOG} check_unit_test kill_server diff --git a/qa/L0_mlflow/plugin_test.py b/qa/L0_mlflow/plugin_test.py index e225ed5881..a5d87a3c19 100755 --- a/qa/L0_mlflow/plugin_test.py +++ b/qa/L0_mlflow/plugin_test.py @@ -1,6 +1,6 @@ #!/usr/bin/python -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -33,9 +33,7 @@ import json import unittest -import mlflow.onnx import numpy as np -import onnx import test_util as tu from mlflow.deployments import get_deploy_client @@ -47,7 +45,7 @@ def setUp(self): def _validate_deployment(self, model_name): # create self.client_.create_deployment( - model_name, f"models:/{model_name}/1", flavor="onnx" + model_name, "models:/{}/1".format(model_name), flavor="onnx" ) # list @@ -81,6 +79,8 @@ def _validate_deployment(self, model_name): def test_onnx_flavor(self): # Log the ONNX model to MLFlow + import mlflow.onnx + import onnx model = onnx.load( "./mlflow-triton-plugin/examples/onnx_float32_int32_int32/1/model.onnx" @@ -92,6 +92,8 @@ def test_onnx_flavor(self): def test_onnx_flavor_with_files(self): # Log the ONNX model and additional Triton config file to MLFlow + import mlflow.onnx + import onnx model = onnx.load( "./mlflow-triton-plugin/examples/onnx_float32_int32_int32/1/model.onnx" @@ -114,65 +116,6 @@ def test_onnx_flavor_with_files(self): filecmp.cmp(config_path, "./models/onnx_model_with_files/config.pbtxt") ) - def test_model_name(self): - EMPTY_MODEL_NAMES = [ - "", - " ", - " ", - "\n", - "\t", - "\r", - "\v", - "\f", - ] - INVALID_PATH_TRAVERSAL_NAMES = [ - "/opt/sys/", - "../../etc/passwd", - "../outside/repo", - "test_models/../identity_py", - "..", - ] - VALID_MODEL_NAMES = [ - "model123", - # "model OAI", TRI-769: Fix this test case - "model.version1", - "...", - "..my_model", - "model..1", - "model....1", - ] - - for model_name in EMPTY_MODEL_NAMES: - model_uri = f"models:/{model_name}/1" - with self.assertRaises(Exception) as e: - self.client_.create_deployment(model_name, model_uri, flavor="onnx") - self.assertIn( - "Model name cannot be empty. Please enter a valid name to deploy.", - str(e.exception), - ) - - for model_name in INVALID_PATH_TRAVERSAL_NAMES: - model_uri = f"models:/{model_name}/1" - with self.assertRaises(Exception) as e: - self.client_.create_deployment(model_name, model_uri, flavor="onnx") - self.assertIn( - f"Path traversal is not allowed in model's name: {model_name}", - str(e.exception), - ) - - for model_name in VALID_MODEL_NAMES: - model = onnx.load( - "./mlflow-triton-plugin/examples/onnx_float32_int32_int32/1/model.onnx" - ) - - # Use a different name to ensure the plugin operates on correct model - mlflow.onnx.log_model( - model, "triton", registered_model_name=f"{model_name}" - ) - - # Validate deployment functionalities - create, list, get, predict, delete - self._validate_deployment(model_name) - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_mlflow/test.sh b/qa/L0_mlflow/test.sh index 5e863a8988..4b5205ba25 100755 --- a/qa/L0_mlflow/test.sh +++ b/qa/L0_mlflow/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -39,11 +39,7 @@ rm -fr *.log *.json # install a higher version of python which uses blinker 1.6, # but it is unknown whether this test should rely on # the default installation of python. - -apt update -qq && apt install python3-venv -y -python3 -m venv .venv - -source .venv/bin/activate +apt remove -y python3-blinker RET=0 @@ -187,7 +183,7 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** Python Test Failed\n***" RET=1 else - check_test_results $TEST_RESULT_FILE 3 + check_test_results $TEST_RESULT_FILE 2 if [ $? -ne 0 ]; then cat $PY_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" @@ -256,7 +252,7 @@ if [ $? -ne 0 ]; then echo -e "\n***\n*** Python Test Failed\n***" RET=1 else - check_test_results $TEST_RESULT_FILE 3 + check_test_results $TEST_RESULT_FILE 2 if [ $? -ne 0 ]; then cat $PY_LOG echo -e "\n***\n*** Test Result Verification Failed\n***" diff --git a/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/expected b/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/expected deleted file mode 100644 index 9e66bbafeb..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/expected +++ /dev/null @@ -1 +0,0 @@ -failed to load 'allocation_strategy_invalid_value' version 1: Invalid argument: Invalid value for 'execution_context_allocation_strategy': 'UNKNOWN' for model instance 'allocation_strategy_invalid_value' diff --git a/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/partial.pbtxt b/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/partial.pbtxt deleted file mode 100644 index 2970c9bf23..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/invalid/allocation_strategy_invalid_value/partial.pbtxt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -parameters: { - key: "execution_context_allocation_strategy" - value: { - string_value: "UNKNOWN" - } -} diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_key/partial.pbtxt b/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_key/partial.pbtxt deleted file mode 100644 index 808432bbc5..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_key/partial.pbtxt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -parameters: {} diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_parameters/partial.pbtxt b/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_parameters/partial.pbtxt deleted file mode 100644 index ce81309581..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_no_parameters/partial.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/expected b/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/expected deleted file mode 100644 index 584d6becdd..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/expected +++ /dev/null @@ -1 +0,0 @@ -'execution_context_allocation_strategy' set to 'STATIC' for model instance 'allocation_strategy_value_1' diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/partial.pbtxt b/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/partial.pbtxt deleted file mode 100644 index ebddccc5b1..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_1/partial.pbtxt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -parameters: { - key: "execution_context_allocation_strategy" - value: { - string_value: "STATIC" - } -} diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/expected b/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/expected deleted file mode 100644 index 3dcd519ddf..0000000000 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/expected +++ /dev/null @@ -1 +0,0 @@ -'execution_context_allocation_strategy' set to 'ON_PROFILE_CHANGE' for model instance 'allocation_strategy_value_2' diff --git a/qa/L0_model_config/test.sh b/qa/L0_model_config/test.sh index 4ef8f76907..7622cf93b2 100755 --- a/qa/L0_model_config/test.sh +++ b/qa/L0_model_config/test.sh @@ -298,7 +298,7 @@ cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/openvino_int8_in cp /data/inferenceserver/${REPO_VERSION}/qa_model_repository/openvino_int8_int8_int8/output0_labels.txt \ autofill_noplatform_success/openvino/partial_config -# Copy decoupled model and config files into the model_metrics test repository. +# Copy decoupled model into the model_metrics test repository. for modelpath in `ls -d model_metrics/*/*`; do src_dir="/opt/tritonserver/qa/python_models/async_execute_decouple" mkdir -p $modelpath/1 @@ -306,18 +306,6 @@ for modelpath in `ls -d model_metrics/*/*`; do cat $src_dir/config.pbtxt $modelpath/partial.pbtxt > $modelpath/config.pbtxt done -# Copy tensorrt model and config files into the custom_parameters test repository. -for modelpath in `ls -d custom_parameters/tensorrt/*/*`; do - mkdir -p $modelpath/1 - model_name=`basename $modelpath` - src_dir="/data/inferenceserver/${REPO_VERSION}/qa_model_repository/plan_float32_float32_float32" - cp ${src_dir}/1/model.plan $modelpath/1/. - cat ${src_dir}/config.pbtxt $modelpath/partial.pbtxt > $modelpath/config.pbtxt - sed -i "s/^name:.*/name: \"${model_name}\"/" $modelpath/config.pbtxt - sed -i "s/^version_policy:.*//" $modelpath/config.pbtxt - sed -i "s/label_filename:.*//" $modelpath/config.pbtxt -done - rm -f $SERVER_LOG_BASE* $CLIENT_LOG RET=0 @@ -706,82 +694,6 @@ for TARGET_DIR in `ls -d model_metrics/invalid_config/*`; do fi done -# Run all custom_parameters tests that are expected to succeed. -for TARGET_DIR in `ls -d custom_parameters/*/valid/*`; do - TARGET_DIR_DOT=`echo $TARGET_DIR | tr / .` - TARGET=`basename ${TARGET_DIR}` - - SERVER_ARGS="--model-repository=`pwd`/models --log-info=true" - SERVER_LOG=$SERVER_LOG_BASE.${TARGET_DIR_DOT}.log - - rm -fr models && mkdir models - cp -r ${TARGET_DIR} models/. - - EXPECTED=models/$TARGET/expected - echo -e "Test $TARGET_DIR" >> $CLIENT_LOG - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "*** FAILED: unable to start $SERVER" >> $CLIENT_LOG - RET=1 - else - kill $SERVER_PID - wait $SERVER_PID - fi - - if [ -f $EXPECTED ]; then - EX_FOUND=0 - EX=`cat $EXPECTED` - if grep ^I[0-9][0-9][0-9][0-9].*"$EX" $SERVER_LOG; then - echo -e "Found \"$EX\"" >> $CLIENT_LOG - EX_FOUND=1 - else - echo -e "Not found \"$EX\"" >> $CLIENT_LOG - fi - if [ "$EX_FOUND" == "0" ]; then - echo -e "*** FAILED: model_metrics/$TARGET" >> $CLIENT_LOG - RET=1 - fi - fi -done - -# Run all custom_parameters tests that have invalid values. -for TARGET_DIR in `ls -d custom_parameters/*/invalid/*`; do - TARGET_DIR_DOT=`echo $TARGET_DIR | tr / .` - TARGET=`basename ${TARGET_DIR}` - - SERVER_ARGS="--model-repository=`pwd`/models --log-info=true" - SERVER_LOG=$SERVER_LOG_BASE.${TARGET_DIR_DOT}.log - - rm -fr models && mkdir models - cp -r ${TARGET_DIR} models/. - - EXPECTED=models/$TARGET/expected - echo -e "Test $TARGET_DIR" >> $CLIENT_LOG - - # We expect all tests to fail with the expected error message - run_server - if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - RET=1 - kill $SERVER_PID - wait $SERVER_PID - else - EX_FOUND=0 - EX=`cat $EXPECTED` - if grep ^E[0-9][0-9][0-9][0-9].*"$EX" $SERVER_LOG; then - echo -e "Found \"$EX\"" >> $CLIENT_LOG - EX_FOUND=1 - else - echo -e "Not found \"$EX\"" >> $CLIENT_LOG - fi - if [ "$EX_FOUND" == "0" ]; then - echo -e "*** FAILED: model_metrics/$TARGET" >> $CLIENT_LOG - RET=1 - fi - fi -done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else diff --git a/qa/L0_openai/generate_engine.py b/qa/L0_openai/generate_engine.py deleted file mode 100644 index 22c21d2209..0000000000 --- a/qa/L0_openai/generate_engine.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from argparse import ArgumentParser - -from tensorrt_llm import BuildConfig -from tensorrt_llm._tensorrt_engine import LLM -from tensorrt_llm.lora_manager import LoraConfig -from tensorrt_llm.plugin import PluginConfig - - -def generate_model_engine(model: str, engines_path: str): - config = BuildConfig(plugin_config=PluginConfig(gemm_plugin="auto")) - - lora_config = LoraConfig( - lora_target_modules=["attn_q", "attn_k", "attn_v"], - max_lora_rank=8, - max_loras=4, - max_cpu_loras=8, - ) - - engine = LLM( - model, - dtype="float16", - max_batch_size=128, - build_config=config, - guided_decoding_backend="xgrammar", - lora_config=lora_config, - ) - - engine.save(engines_path) - engine.shutdown() - - -if __name__ == "__main__": - parser = ArgumentParser() - parser.add_argument( - "--model", "-m", help="model huggingface id or path to the model" - ) - parser.add_argument("--engine_path", "-e", help="directory of the output engine") - FLAGS = parser.parse_args() - - generate_model_engine(FLAGS.model, FLAGS.engine_path) - print(f"model {FLAGS.model}'s engine has been saved to {FLAGS.engine_path}") diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index 5fd4daf239..0921bce98e 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,16 +27,6 @@ ### Helpers ### -function download_tensorrt_llm_models { - TENSORRTLLM_DIR="$1" - rm -rf ${TENSORRTLLM_DIR} && mkdir ${TENSORRTLLM_DIR} - git clone --filter=blob:none --no-checkout https://github.com/triton-inference-server/TensorRT-LLM.git ${TENSORRTLLM_DIR} - pushd ${TENSORRTLLM_DIR} - git sparse-checkout set triton_backend/all_models - git checkout ${TENSORRTLLM_REPO_TAG} - popd -} - function install_deps() { # Install python bindings for tritonserver and tritonfrontend # pip install /opt/tritonserver/python/triton*.whl @@ -48,12 +38,7 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then - # TODO: Remove this when the next stable version of TRT-LLM is available - TENSORRTLLM_DIR="/workspace/TensorRT-LLM" - download_tensorrt_llm_models ${TENSORRTLLM_DIR} - - prepare_tensorrtllm meta-llama/Meta-Llama-3.1-8B-Instruct tests/tensorrtllm_models /tmp/engines/llama/3.1-8b-instruct/ ${TENSORRTLLM_DIR} - prepare_tensorrtllm mistralai/Mistral-Nemo-Instruct-2407 tests/tensorrtllm_mistral_models /tmp/engines/mistral/nemo-instruct-2407/ ${TENSORRTLLM_DIR} + prepare_tensorrtllm else prepare_vllm fi @@ -68,46 +53,38 @@ function prepare_tensorrtllm() { # FIXME: Remove when testing TRT-LLM containers built from source pip install -r requirements.txt - MODEL="$1" - MODEL_REPO="$2" - ENGINE_PATH="$3" - TENSORRTLLM_DIR="$4" - TRITON_BACKEND=tensorrtllm - XGRAMMAR_TOKENIZER_INFO_PATH=tokenizer_info/${MODEL}/xgrammar_tokenizer_info.json - GUIDED_DECODING_BACKEND=xgrammar - + MODEL="meta-llama/Meta-Llama-3.1-8B-Instruct" + MODEL_REPO="tests/tensorrtllm_models" mkdir -p ${MODEL_REPO} - cp ${TENSORRTLLM_DIR}/triton_backend/all_models/inflight_batcher_llm/* "${MODEL_REPO}" -r + cp /app/all_models/inflight_batcher_llm/* "${MODEL_REPO}" -r # Ensemble model is not needed for the test rm -rf ${MODEL_REPO}/ensemble - # 1. Generate the model's trt engines - python3 ../generate_engine.py --model "${MODEL}" --engine_path "${ENGINE_PATH}" + # 1. Download model from HF + huggingface-cli download ${MODEL} - # 2. Generate the model's xgrammar tokenizer info. In order to run on C++ backend, we need an extra step to extract tokenizer’s information into json format. - XGRAMMAR_TOKENIZER_INFO_DIR=tokenizer_info/${MODEL} - rm -rf ${XGRAMMAR_TOKENIZER_INFO_DIR} - python3 /app/examples/generate_xgrammar_tokenizer_info.py --model_dir ${MODEL} --output_dir ${XGRAMMAR_TOKENIZER_INFO_DIR} + HF_LLAMA_MODEL=`python3 -c "from pathlib import Path; from huggingface_hub import hf_hub_download; print(Path(hf_hub_download('${MODEL}', filename='config.json')).parent)"` + CKPT_PATH=/tmp/ckpt/llama/3.1-8b-instruct/ + ENGINE_PATH=/tmp/engines/llama/3.1-8b-instruct/ - # 3. Prepare model repository + # 2. Convert weights + python3 /app/examples/llama/convert_checkpoint.py --model_dir ${HF_LLAMA_MODEL} \ + --output_dir ${CKPT_PATH} \ + --dtype float16 + + # 3. Build engine + # max_batch_size set to 128 to avoid OOM errors + trtllm-build --checkpoint_dir ${CKPT_PATH} \ + --gemm_plugin auto \ + --max_batch_size 128 \ + --output_dir ${ENGINE_PATH} + + # 4. Prepare model repository FILL_TEMPLATE="/app/tools/fill_template.py" - python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/preprocessing/config.pbtxt tokenizer_dir:${ENGINE_PATH},triton_max_batch_size:64,preprocessing_instance_count:1,max_queue_size:0 - python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/postprocessing/config.pbtxt tokenizer_dir:${ENGINE_PATH},triton_max_batch_size:64,postprocessing_instance_count:1 - python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm_bls/config.pbtxt triton_max_batch_size:64,decoupled_mode:True,bls_instance_count:1,accumulate_tokens:False,logits_datatype:TYPE_FP32,prompt_embedding_table_data_type:TYPE_FP16 - python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm/config.pbtxt triton_backend:${TRITON_BACKEND},triton_max_batch_size:64,decoupled_mode:True,max_beam_width:1,engine_dir:${ENGINE_PATH},batching_strategy:inflight_fused_batching,max_queue_size:0,max_queue_delay_microseconds:1000,encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32,exclude_input_in_output:True,prompt_embedding_table_data_type:TYPE_FP16,guided_decoding_backend:${GUIDED_DECODING_BACKEND},xgrammar_tokenizer_info_path:${XGRAMMAR_TOKENIZER_INFO_PATH} - - # 4. Prepare lora adapters - # FIXME: Remove this WAR when it is fixed in the future stable version of TRT-LLM. - sed -i 's/dims: \[ -1, 3 \]/dims: \[ -1, 4 \]/' ${MODEL_REPO}/tensorrt_llm/config.pbtxt - sed -i 's/dims: \[ -1, 3 \]/dims: \[ -1, 4 \]/' ${MODEL_REPO}/tensorrt_llm_bls/config.pbtxt - pushd ${MODEL_REPO}/tensorrt_llm_bls/1 - for lora_name in silk-road/luotuo-lora-7b-0.1 kunishou/Japanese-Alpaca-LoRA-7b-v0; do - name=$(basename $lora_name) - git clone https://huggingface.co/$lora_name - python3 /app/examples/hf_lora_convert.py -i $name -o $name-weights --storage-type float16 - rm -rf $name - done - popd + python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/preprocessing/config.pbtxt tokenizer_dir:${HF_LLAMA_MODEL},triton_max_batch_size:64,preprocessing_instance_count:1,max_queue_size:0 + python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/postprocessing/config.pbtxt tokenizer_dir:${HF_LLAMA_MODEL},triton_max_batch_size:64,postprocessing_instance_count:1 + python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm_bls/config.pbtxt triton_max_batch_size:64,decoupled_mode:True,bls_instance_count:1,accumulate_tokens:False,logits_datatype:TYPE_FP32 + python3 ${FILL_TEMPLATE} -i ${MODEL_REPO}/tensorrt_llm/config.pbtxt triton_backend:tensorrtllm,triton_max_batch_size:64,decoupled_mode:True,max_beam_width:1,engine_dir:${ENGINE_PATH},batching_strategy:inflight_fused_batching,max_queue_size:0,max_queue_delay_microseconds:1000,encoder_input_features_data_type:TYPE_FP16,logits_datatype:TYPE_FP32,exclude_input_in_output:True } function pre_test() { @@ -123,30 +100,17 @@ function pre_test() { function run_test() { pushd openai/ TEST_LOG="test_openai.log" - TEST_XML="test_openai.xml" - TEST_LOG_MISTRAL="test_openai_mistral.log" - TEST_XML_MISTRAL="test_openai_mistral.xml" + # Capture error code without exiting to allow log collection set +e - pytest -s -v --junitxml=${TEST_XML} tests/ 2>&1 | tee ${TEST_LOG} - if [ ${PIPESTATUS[0]} -ne 0 ]; then + pytest -s -v --junitxml=test_openai.xml tests/ 2>&1 > ${TEST_LOG} + if [ $? -ne 0 ]; then + cat ${TEST_LOG} echo -e "\n***\n*** Test Failed\n***" RET=1 fi set -e - if [ "$RET" == "0" ]; then - # rerun the tool calling tests with mistral model to cover the mistral tool call parser - set +e - TEST_TOOL_CALL_PARSER="mistral" TEST_TOKENIZER="mistralai/Mistral-Nemo-Instruct-2407" \ - pytest -s -v --junitxml=${TEST_XML_MISTRAL} tests/test_tool_calling.py 2>&1 | tee ${TEST_LOG_MISTRAL} - if [ ${PIPESTATUS[0]} -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - set -e - fi - # Collect logs for error analysis when needed cp *.xml *.log ../../../ popd diff --git a/qa/L0_orca/test.sh b/qa/L0_orca/test.sh index 50b7f450fb..6069a75048 100755 --- a/qa/L0_orca/test.sh +++ b/qa/L0_orca/test.sh @@ -36,7 +36,7 @@ MODEL_NAME="gpt2_tensorrt_llm" NAME="tensorrt_llm_benchmarking_test" MODEL_REPOSITORY="$(pwd)/triton_model_repo" TENSORRTLLM_BACKEND_DIR="/workspace/tensorrtllm_backend" -GPT_DIR="$TENSORRTLLM_BACKEND_DIR/tensorrt_llm/examples/models/core/gpt" +GPT_DIR="$TENSORRTLLM_BACKEND_DIR/tensorrt_llm/examples/gpt" TOKENIZER_DIR="$GPT_DIR/gpt2" ENGINES_DIR="${BASE_DIR}/engines/inflight_batcher_llm/${NUM_GPUS}-gpu" TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} @@ -46,7 +46,97 @@ SERVER_LOG="${NAME}_server.log" SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} CLIENT_PY=${BASE_DIR}/orca_http_test.py CLIENT_LOG="${NAME}_orca_http_test.log" -source ../common/trtllm_util.sh +source ../common/util.sh + +function prepare_model_repository { + rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} + cp -r ${TENSORRTLLM_BACKEND_DIR}/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} + rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls + mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" + + replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" +} + +# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on +# success, 1 on failure +function wait_for_server_ready() { + local wait_time_secs="${1:-30}" + shift + local spids=("$@") + + WAIT_RET=0 + + for _ in $(seq "$wait_time_secs"); do + for pid in "${spids[@]}"; do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "=== Server not running." + WAIT_RET=1 + return + fi + done + + sleep 1 + + if curl -s --fail localhost:8000/v2/health/ready && + curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then + return + fi + done + + echo "=== Timeout $wait_time_secs secs. Server not ready." + WAIT_RET=1 +} + +function run_server { + python3 ${TENSORRTLLM_BACKEND_DIR}/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & + sleep 2 # allow time to obtain the pid(s) + # Read PIDs into an array, trimming whitespaces + readarray -t SERVER_PID < <(pgrep "tritonserver") + + wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" + if [ "$WAIT_RET" != "0" ]; then + # Cleanup + kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi +} + +function kill_server { + pgrep tritonserver | xargs kill -SIGINT + for pid in "${SERVER_PID[@]}"; do + echo "Waiting for proc ${pid} to terminate..." + while kill -0 $pid >/dev/null 2>&1; do + sleep 1 + done + done +} clone_tensorrt_llm_backend_repo build_gpt2_base_model diff --git a/qa/L0_parameters/class_count_test.py b/qa/L0_parameters/class_count_test.py deleted file mode 100755 index 801c442b87..0000000000 --- a/qa/L0_parameters/class_count_test.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import os -import unittest - -import numpy as np -import test_util as tu -import tritonclient.grpc as grpcclient -import tritonclient.http as httpclient -from tritonclient.utils import InferenceServerException - - -class ClassificationParameterTest(tu.TestResultCollector): - def setUp(self): - self.protocol = os.environ.get("CLIENT_TYPE", "http") - if self.protocol == "http": - self.client = httpclient.InferenceServerClient("localhost:8000") - else: - self.client = grpcclient.InferenceServerClient("localhost:8001") - - def _prepare_io(self, input_data, dtype): - if self.protocol == "http": - inputs = [httpclient.InferInput("INPUT0", input_data.shape, dtype)] - outputs = [httpclient.InferRequestedOutput(name="OUTPUT0", class_count=5)] - else: - inputs = [grpcclient.InferInput("INPUT0", input_data.shape, dtype)] - outputs = [grpcclient.InferRequestedOutput(name="OUTPUT0", class_count=5)] - inputs[0].set_data_from_numpy(input_data) - return inputs, outputs - - def test_classificattion(self): - shape = (1, 8) - dtype = "FP32" - model_name = "identity_fp32" - input_data = np.ones(shape, dtype=np.float32) - - inputs, outputs = self._prepare_io(input_data, dtype) - result = self.client.infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - output = result.get_output("OUTPUT0") - if self.protocol == "http": - output_dtype = output["datatype"] - else: - output_dtype = output.datatype - - self.assertEqual(output_dtype, "BYTES") - - # Validate shape matches to the class_count - output_data = result.as_numpy("OUTPUT0") - self.assertIsNotNone(output_data) - self.assertEqual(output_data.shape, (1, 5)) - - for res_str_bytes in np.nditer(output_data, flags=["refs_ok"]): - res_str = res_str_bytes.item().decode("utf-8") - self.assertTrue(res_str.startswith("1.000000:")) - - def test_classificattion_unsupported_data_type(self): - shape = (1, 8) - model_name = "identity_bytes" - dtype = "BYTES" - input_data = np.array([["test"] * shape[1]], dtype=object) - - inputs, outputs = self._prepare_io(input_data, dtype) - with self.assertRaises(InferenceServerException) as e: - self.client.infer(model_name=model_name, inputs=inputs, outputs=outputs) - - self.assertIn( - "class result not available for output due to unsupported type 'BYTES'", - str(e.exception), - ) - - def test_classification_output_tensor_too_large(self): - max_elements = 1_000_000 - shape = (1, max_elements + 1) - dtype = "FP32" - model_name = "identity_fp32" - input_data = np.ones(shape, dtype=np.float32) - - inputs, outputs = self._prepare_io(input_data, dtype) - with self.assertRaises(InferenceServerException) as e: - self.client.infer(model_name=model_name, inputs=inputs, outputs=outputs) - - self.assertIn("classification output tensor too large", str(e.exception)) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_parameters/parameters_test.py b/qa/L0_parameters/parameters_test.py index 90221c12d7..a20d13c1eb 100755 --- a/qa/L0_parameters/parameters_test.py +++ b/qa/L0_parameters/parameters_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -30,86 +30,66 @@ sys.path.append("../common") +import os import queue import unittest from functools import partial +from unittest import IsolatedAsyncioTestCase import numpy as np -import requests import tritonclient.grpc as grpcclient import tritonclient.grpc.aio as asyncgrpcclient import tritonclient.http as httpclient import tritonclient.http.aio as asynchttpclient from tritonclient.utils import InferenceServerException -_TRITON_RESERVED_CLIENT_ERROR = "is a reserved parameter and cannot be specified" -_TRITON_RESERVED_SERVER_ERROR = "reserved for Triton usage" - -# docs/protocol/extension_parameters.md — reserved names -_RESERVED_PARAMETER_KEYS = ( - "sequence_id", - "sequence_start", - "sequence_end", - "priority", - "timeout", - "headers", - "binary_data_output", - "triton_enable_empty_final_response", - "triton_final_response", - "triton_injected_via_header", -) - -_HTTP_LOCALHOST = "http://localhost:8000" - - -def _infer_url(model_name="parameter"): - return f"{_HTTP_LOCALHOST}/v2/models/{model_name}/infer" - - -def _minimal_fp32_infer_body(parameters=None): - """KServe HTTP infer JSON matching qa/L0_parameters model `parameter` (INPUT0 FP32 [1]).""" - body = { - "inputs": [ - { - "name": "INPUT0", - "shape": [1], - "datatype": "FP32", - "data": [1.0], - } - ], - } - if parameters is not None: - body["parameters"] = parameters - return body - +TEST_HEADER = os.environ.get("TEST_HEADER") -_FORWARD_HEADERS = { - "header_1": "value_1", - "header_2": "value_2", - "my_header_1": "my_value_1", - "my_header_2": "my_value_2", - "my_header_3": 'This is a "quoted" string with a backslash\\ ', -} +class InferenceParametersTest(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.http = httpclient.InferenceServerClient(url="localhost:8000") + self.async_http = asynchttpclient.InferenceServerClient(url="localhost:8000") + self.grpc = grpcclient.InferenceServerClient(url="localhost:8001") + self.async_grpc = asyncgrpcclient.InferenceServerClient(url="localhost:8001") + + self.parameter_list = [] + self.parameter_list.append({"key1": "value1", "key2": "value2"}) + self.parameter_list.append({"key1": 1, "key2": 2}) + self.parameter_list.append({"key1": 123.123, "key2": 321.321}) + self.parameter_list.append({"key1": True, "key2": "value2"}) + self.parameter_list.append({"triton_": True, "key2": "value2"}) + + # Only "test_params" tests parameters without headers. + if TEST_HEADER != "test_params": + self.headers = { + "header_1": "value_1", + "header_2": "value_2", + "my_header_1": "my_value_1", + "my_header_2": "my_value_2", + "my_header_3": 'This is a "quoted" string with a backslash\ ', + } -def _grpc_stream_callback(user_data, result, error): - if error: - user_data.put(error) - else: - user_data.put(result) + # only these headers should be forwarded to the model. + if TEST_HEADER == "test_grpc_header_forward_pattern_case_sensitive": + self.expected_headers = {} + else: + self.expected_headers = { + "my_header_1": "my_value_1", + "my_header_2": "my_value_2", + "my_header_3": 'This is a "quoted" string with a backslash\ ', + } + else: + self.headers = {} + self.expected_headers = {} + def callback(user_data, result, error): + if error: + user_data.put(error) + else: + user_data.put(result) -class InferenceParametersTest(unittest.IsolatedAsyncioTestCase): - async def asyncSetUp(self): - self.httpclient = httpclient.InferenceServerClient(url="localhost:8000") - self.async_httpclient = asynchttpclient.InferenceServerClient( - url="localhost:8000" - ) - self.grpcclient = grpcclient.InferenceServerClient(url="localhost:8001") - self.async_grpcclient = asyncgrpcclient.InferenceServerClient( - url="localhost:8001" - ) - self.grpcclient_callback = _grpc_stream_callback + self.grpc_callback = callback def create_inputs(self, client_type): inputs = [] @@ -120,257 +100,128 @@ def create_inputs(self, client_type): return inputs async def send_request_and_verify( - self, - client_type, - client, - parameters, - headers, - expected_headers, - is_async_infer=False, - model_name="parameter", + self, client_type, client, is_async=False, model_name="parameter" ): inputs = self.create_inputs(client_type) - - if is_async_infer: - if client_type == httpclient: - result = client.async_infer( - model_name=model_name, - inputs=inputs, - parameters=parameters, - headers=headers, - ).get_result() - elif client_type == grpcclient: - user_data = queue.Queue() - client.async_infer( - model_name=model_name, - inputs=inputs, - parameters=parameters, - headers=headers, - callback=partial(self.grpcclient_callback, user_data), - ) - result = user_data.get() - self.assertIsNot(result, InferenceServerException) - else: - raise ValueError(f"Unsupported client type: {client_type}") - else: + for parameters in self.parameter_list: + # Setup infer callable to re-use below for brevity infer_callable = partial( client.infer, model_name=model_name, inputs=inputs, parameters=parameters, - headers=headers, + headers=self.headers, ) - if client_type == asynchttpclient or client_type == asyncgrpcclient: - result = await infer_callable() + + # The `triton_` prefix is reserved for Triton usage + should_error = False + if "triton_" in parameters.keys(): + should_error = True + + if is_async: + if should_error: + with self.assertRaises(InferenceServerException): + await infer_callable() + return + else: + result = await infer_callable() else: - result = infer_callable() + if should_error: + with self.assertRaises(InferenceServerException): + infer_callable() + return + else: + result = infer_callable() - self.verify_outputs(result, parameters, expected_headers) + self.verify_outputs(result, parameters) - def verify_outputs(self, result, parameters, expected_headers): + def verify_outputs(self, result, parameters): keys = result.as_numpy("key") values = result.as_numpy("value") keys = keys.astype(str).tolist() - expected_keys = list(parameters.keys()) + list(expected_headers.keys()) - self.assertEqual( - set(keys), - set(expected_keys), - msg=f"keys: {keys}, expected_keys: {expected_keys}", - ) + expected_keys = list(parameters.keys()) + list(self.expected_headers.keys()) + self.assertEqual(set(keys), set(expected_keys)) # We have to convert the parameter values to string expected_values = [] for expected_value in list(parameters.values()): expected_values.append(str(expected_value)) - for value in expected_headers.values(): + for value in self.expected_headers.values(): expected_values.append(value) - self.assertEqual( - set(values.astype(str).tolist()), - set(expected_values), - msg=f"values: {values.astype(str).tolist()}, expected_values: {expected_values}", - ) + self.assertEqual(set(values.astype(str).tolist()), set(expected_values)) - async def _verify_grpc_stream_infer(self, parameters, headers, expected_headers): - user_data = queue.Queue() - self.grpcclient.start_stream( - callback=partial(self.grpcclient_callback, user_data), headers=headers - ) - inputs = self.create_inputs(grpcclient) - self.grpcclient.async_stream_infer( - model_name="parameter", inputs=inputs, parameters=parameters - ) - result = user_data.get() - self.assertIsNot(result, InferenceServerException) - self.verify_outputs(result, parameters, expected_headers) - self.grpcclient.stop_stream() - - def _raw_http_post_infer(self, body_dict, extra_headers=None): - """POST /v2/models/.../infer without tritonclient (no header normalization).""" - headers = {"Content-Type": "application/json"} - if extra_headers: - headers.update(extra_headers) - return requests.post( - _infer_url("parameter"), - json=body_dict, - headers=headers, - timeout=60, - ) + async def test_grpc_parameter(self): + await self.send_request_and_verify(grpcclient, self.grpc) - def _assert_raw_http_400(self, response, text_substr): - """Assert status 400 and body contains `text_substr`.""" - snippet = response.text[:2000] if response.text else "" - self.assertEqual(response.status_code, 400, msg=snippet) - self.assertIn(text_substr, response.text, msg=snippet) + async def test_http_parameter(self): + await self.send_request_and_verify(httpclient, self.http) - async def _run_client_infer_suite(self, parameters, headers, expected_headers): - """ - Full client matrix: gRPC/HTTP sync+async, stream, ensemble. - """ - await self.send_request_and_verify( - grpcclient, self.grpcclient, parameters, headers.copy(), expected_headers - ) + async def test_async_http_parameter(self): await self.send_request_and_verify( - httpclient, self.httpclient, parameters, headers.copy(), expected_headers - ) - await self.send_request_and_verify( - asynchttpclient, - self.async_httpclient, - parameters, - headers.copy(), - expected_headers, - ) - await self.send_request_and_verify( - asyncgrpcclient, - self.async_grpcclient, - parameters, - headers.copy(), - expected_headers, - ) - await self.send_request_and_verify( - httpclient, - self.httpclient, - parameters, - headers.copy(), - expected_headers, - is_async_infer=True, - ) - await self.send_request_and_verify( - grpcclient, - self.grpcclient, - parameters, - headers.copy(), - expected_headers, - is_async_infer=True, - ) - await self._verify_grpc_stream_infer( - parameters, headers.copy(), expected_headers + asynchttpclient, self.async_http, is_async=True ) + + async def test_async_grpc_parameter(self): await self.send_request_and_verify( - httpclient, - self.httpclient, - parameters, - headers.copy(), - expected_headers, - model_name="ensemble", + asyncgrpcclient, self.async_grpc, is_async=True ) - async def test_params(self): - for parameters in [ - {"key1": "value1", "key2": "value2"}, - {"key1": 1, "key2": 2}, - {"key1": 123.123, "key2": 321.321}, - {"key1": True, "key2": "value2"}, - ]: - await self._run_client_infer_suite(parameters, {}, {}) - - async def test_params_reserved_rejected(self): - for reserved_key in _RESERVED_PARAMETER_KEYS: - parameters = {reserved_key: "dummy-value"} - body = _minimal_fp32_infer_body(parameters) - - # Raw HTTP - if reserved_key.startswith("triton_"): - r = self._raw_http_post_infer(body) - self._assert_raw_http_400(r, _TRITON_RESERVED_SERVER_ERROR) - - # Python clients - for client_type, client in ( - (httpclient, self.httpclient), - (asynchttpclient, self.async_httpclient), - (grpcclient, self.grpcclient), - (asyncgrpcclient, self.async_grpcclient), - ): - inputs = self.create_inputs(client_type) - with self.assertRaises(InferenceServerException) as cm: - if client_type in (asynchttpclient, asyncgrpcclient): - await client.infer( - model_name="parameter", - inputs=inputs, - parameters=parameters, - ) - else: - client.infer( - model_name="parameter", - inputs=inputs, - parameters=parameters, - ) - msg = str(cm.exception) - # Reserved parameters are rejected by the client - self.assertIn(_TRITON_RESERVED_CLIENT_ERROR, msg, msg=msg) - - async def test_headers(self): - expected_headers = { - "my_header_1": "my_value_1", - "my_header_2": "my_value_2", - "my_header_3": 'This is a "quoted" string with a backslash\\ ', - } - await self._run_client_infer_suite({}, _FORWARD_HEADERS, expected_headers) - - async def test_grpc_header_forward_pattern_case_sensitive(self): - expected_headers = {} - await self._run_client_infer_suite({}, _FORWARD_HEADERS, expected_headers) - - async def test_headers_reserved_rejected(self): - body = _minimal_fp32_infer_body({}) - for reserved_key in _RESERVED_PARAMETER_KEYS: - # Raw HTTP - r = self._raw_http_post_infer( - body, extra_headers={reserved_key: "dummy-value"} + def test_http_async_parameter(self): + inputs = self.create_inputs(httpclient) + # Skip the parameter that returns an error + parameter_list = self.parameter_list[:-1] + for parameters in parameter_list: + result = self.http.async_infer( + model_name="parameter", + inputs=inputs, + parameters=parameters, + headers=self.headers, + ).get_result() + self.verify_outputs(result, parameters) + + def test_grpc_async_parameter(self): + user_data = queue.Queue() + inputs = self.create_inputs(grpcclient) + # Skip the parameter that returns an error + parameter_list = self.parameter_list[:-1] + for parameters in parameter_list: + self.grpc.async_infer( + model_name="parameter", + inputs=inputs, + parameters=parameters, + headers=self.headers, + callback=partial(self.grpc_callback, user_data), ) - self._assert_raw_http_400(r, _TRITON_RESERVED_SERVER_ERROR) - - # Python clients - for client_type, client in ( - (httpclient, self.httpclient), - (asynchttpclient, self.async_httpclient), - (grpcclient, self.grpcclient), - (asyncgrpcclient, self.async_grpcclient), - ): - inputs = self.create_inputs(client_type) - with self.assertRaises(InferenceServerException) as cm: - if client_type in (asynchttpclient, asyncgrpcclient): - await client.infer( - model_name="parameter", - inputs=inputs, - parameters={}, - headers={reserved_key: "dummy-value"}, - ) - else: - client.infer( - model_name="parameter", - inputs=inputs, - parameters={}, - headers={reserved_key: "dummy-value"}, - ) - msg = str(cm.exception) - # Headers are not rejected by the client - self.assertIn(_TRITON_RESERVED_SERVER_ERROR, msg, msg=msg) + result = user_data.get() + self.assertFalse(result is InferenceServerException) + self.verify_outputs(result, parameters) + + def test_grpc_stream_parameter(self): + user_data = queue.Queue() + self.grpc.start_stream( + callback=partial(self.grpc_callback, user_data), headers=self.headers + ) + inputs = self.create_inputs(grpcclient) + # Skip the parameter that returns an error + parameter_list = self.parameter_list[:-1] + for parameters in parameter_list: + # async stream infer + self.grpc.async_stream_infer( + model_name="parameter", inputs=inputs, parameters=parameters + ) + result = user_data.get() + self.assertFalse(result is InferenceServerException) + self.verify_outputs(result, parameters) + self.grpc.stop_stream() + + async def test_ensemble_parameter_forwarding(self): + await self.send_request_and_verify(httpclient, self.http, model_name="ensemble") async def asyncTearDown(self): - self.httpclient.close() - self.grpcclient.close() - await self.async_grpcclient.close() - await self.async_httpclient.close() + self.http.close() + self.grpc.close() + await self.async_grpc.close() + await self.async_http.close() if __name__ == "__main__": diff --git a/qa/L0_parameters/test.sh b/qa/L0_parameters/test.sh index a6ca428c4b..c53b02d4b7 100755 --- a/qa/L0_parameters/test.sh +++ b/qa/L0_parameters/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -39,6 +39,8 @@ if [ ! -z "$TEST_REPO_ARCH" ]; then fi CLIENT_LOG="./client.log" +TEST_SCRIPT_PY="parameters_test.py" + SERVER=/opt/tritonserver/bin/tritonserver SERVER_LOG="./inference_server.log" source ../common/util.sh @@ -52,27 +54,27 @@ mkdir -p "${MODELDIR}/ensemble/1" # https://jirasw.nvidia.com/browse/DLIS-4673 all_tests=("test_params" - "test_params_reserved_rejected" "test_headers" - "test_grpc_header_forward_pattern_case_sensitive" - "test_headers_reserved_rejected") + "test_header_forward_pattern_case_insensitive" + "test_grpc_header_forward_pattern_case_sensitive") RET=0 -for test in "${all_tests[@]}"; do +for i in "${all_tests[@]}"; do + # TEST_HEADER is a parameter used by `parameters_test.py` that controls + # whether the script will test for inclusion of headers in parameters or not. SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120" - if [ "$test" == "test_headers" ]; then - # gRPC lowercases metadata keys; HTTP may lowercase too — match prefix case-insensitively. + if [ "$i" == "test_headers" ]; then + SERVER_ARGS+=" --grpc-header-forward-pattern my_header.*" + SERVER_ARGS+=" --http-header-forward-pattern my_header.*" + elif [ "$i" == "test_header_forward_pattern_case_insensitive" ]; then SERVER_ARGS+=" --grpc-header-forward-pattern MY_HEADER.*" SERVER_ARGS+=" --http-header-forward-pattern MY_HEADER.*" # NOTE: headers sent through the python HTTP client may be automatically # lowercased by internal libraries like geventhttpclient, so we only test # GRPC client for case-sensitivity here: # https://github.com/geventhttpclient/geventhttpclient/blob/d1e14356c3b02099c879cf9b3bdb684a0cbd8bf5/src/geventhttpclient/header.py#L62-L63 - elif [ "$test" == "test_grpc_header_forward_pattern_case_sensitive" ]; then + elif [ "$i" == "test_grpc_header_forward_pattern_case_sensitive" ]; then SERVER_ARGS+=" --grpc-header-forward-pattern (?-i)MY_HEADER.*" - elif [ "$test" == "test_headers_reserved_rejected" ]; then - SERVER_ARGS+=" --grpc-header-forward-pattern .*" - SERVER_ARGS+=" --http-header-forward-pattern .*" fi run_server if [ "$SERVER_PID" == "0" ]; then @@ -82,7 +84,7 @@ for test in "${all_tests[@]}"; do fi set +e - python3 -m unittest "parameters_test.InferenceParametersTest.${test}" >$CLIENT_LOG 2>&1 + TEST_HEADER="$i" python3 $TEST_SCRIPT_PY >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then cat $CLIENT_LOG echo -e "\n***\n*** Test Failed\n***" @@ -95,65 +97,10 @@ for test in "${all_tests[@]}"; do wait $SERVER_PID done - -# Test Classification Extension -PYTHON_MODELS_DIR="${PYTHON_MODELS_DIR:-/opt/tritonserver/qa/python_models}" -MODELDIR="models" -TEST_RESULT_FILE="test_results.txt" -TEST_SCRIPT_PY="./class_count_test.py" - -rm -rf $MODELDIR -mkdir -p "${MODELDIR}/identity_fp32/1" -cp ${PYTHON_MODELS_DIR}/identity_fp32/config.pbtxt "${MODELDIR}/identity_fp32/" -cp ${PYTHON_MODELS_DIR}/identity_fp32/model.py "${MODELDIR}/identity_fp32/1/" - -mkdir -p "${MODELDIR}/identity_bytes/1" -cp ${PYTHON_MODELS_DIR}/identity_fp32/config.pbtxt "${MODELDIR}/identity_bytes/" -cp ${PYTHON_MODELS_DIR}/identity_fp32/model.py "${MODELDIR}/identity_bytes/1/" -(cd "${MODELDIR}/identity_bytes" && \ - sed -i 's/identity_fp32/identity_bytes/' config.pbtxt && \ - sed -i 's/TYPE_FP32/TYPE_STRING/' config.pbtxt ) - -SERVER_ARGS="--model-repository=`pwd`/${MODELDIR} --log-verbose=1" -for client_type in http grpc; do - export CLIENT_TYPE=$client_type - SERVER_LOG="./class_count_test_${client_type}_server.log" - CLIENT_LOG="./class_count_test_${client_type}_client.log" - rm -f $SERVER_LOG $CLIENT_LOG - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - python3 $TEST_SCRIPT_PY -v >>"$CLIENT_LOG" 2>&1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed - class_count_${client_type}_test_client\n***" - RET=1 - else - check_test_results $TEST_RESULT_FILE 3 - if [ $? -ne 0 ]; then - cat $TEST_RESULT_FILE - echo -e "\n***\n*** Test Result Verification Failed - class_count_${client_type}_test_client\n***" - RET=1 - fi - fi - kill $SERVER_PID - wait $SERVER_PID - - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Server shut down non-gracefully\n***" - RET=1 - fi - set -e -done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else + cat $CLIENT_LOG echo -e "\n***\n*** Test FAILED\n***" fi diff --git a/qa/L0_passive_instance/test.sh b/qa/L0_passive_instance/test.sh index be56bb4185..8948434485 100755 --- a/qa/L0_passive_instance/test.sh +++ b/qa/L0_passive_instance/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,8 +45,7 @@ CLIENT_LOG="./client.log" TEST_SCRIPT_PY=passive_instance_test.py EXPECTED_NUM_TESTS="1" -pip3 install perf_analyzer -PERF_ANALYZER=perf_analyzer +PERF_ANALYZER=../clients/perf_analyzer MODEL=distributed_int32_int32_int32 SERVER=/opt/tritonserver/bin/tritonserver diff --git a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/partial.pbtxt b/qa/L0_perf_analyzer/nginx.conf similarity index 81% rename from qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/partial.pbtxt rename to qa/L0_perf_analyzer/nginx.conf index 11a75b4e9d..4a7dfcc04a 100644 --- a/qa/L0_model_config/custom_parameters/tensorrt/valid/allocation_strategy_value_2/partial.pbtxt +++ b/qa/L0_perf_analyzer/nginx.conf @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -23,9 +23,16 @@ # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -parameters: { - key: "execution_context_allocation_strategy" - value: { - string_value: "ON_PROFILE_CHANGE" - } + +server { + listen 443 ssl; + server_name localhost; + + ssl_certificate /etc/nginx/cert.crt; + ssl_certificate_key /etc/nginx/cert.key; + + location / { + proxy_pass http://localhost:8000; + proxy_http_version 1.1; + } } diff --git a/qa/L0_perf_analyzer/perf_analyzer_profile_export_schema.json b/qa/L0_perf_analyzer/perf_analyzer_profile_export_schema.json new file mode 100644 index 0000000000..d0feacd9b4 --- /dev/null +++ b/qa/L0_perf_analyzer/perf_analyzer_profile_export_schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/perf_analyzer/examples/schema.json", + "title": "Perf Analyzer output data", + "description": "A json file describing the output from a Perf Analyzer run.", + "type": "object", + "required": [ + "experiments", + "version" + ], + "properties": { + "experiments": { + "description": "The array of all experiments run by Perf Analyzer.", + "type": "array", + "required": [ + "experiment", + "requests", + "window_boundaries" + ], + "minItems": 1, + "uniqueItems": true, + "items": { + "type": "object", + "properties": { + "experiment": { + "description": "A single experiment run by Perf Analyzer.", + "type": "object", + "required": [ + "mode", + "value" + ], + "minItems": 1, + "maxItems": 1, + "properties": { + "mode": { + "description": "Operating mode of Perf Analyzer: For example, 'concurrency' or 'request rate'.", + "type": "string" + }, + "value": { + "description": "Concurrency or request rate for the current experiment.", + "type": "integer" + } + } + }, + "requests": { + "description": "The array of requests sent by Perf Analyzer for this experiment.", + "type": "array", + "items": { + "$ref": "#/properties/experiments/items/properties/$defs/request" + } + }, + "$defs": { + "request": { + "description": "Info for a single request.", + "type": "object", + "required": [ + "timestamp", + "response_timestamps" + ], + "properties": { + "timestamp": { + "description": "Time stamp of the request.", + "type": "integer" + }, + "sequence_id": { + "description": "The sequence_id of the request.", + "type": "integer" + }, + "response_timestamps": { + "description": "All associated responses to this request.", + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + "window_boundaries": { + "description": "An array of time stamps describing window boundaries.", + "type": "array", + "items": { + "type": "integer" + }, + "uniqueItems": true + } + } + } + }, + "version": { + "description": "The version of Perf Analyzer that generated the report.", + "type": "string" + } + } +} \ No newline at end of file diff --git a/qa/L0_perf_analyzer/test.sh b/qa/L0_perf_analyzer/test.sh new file mode 100755 index 0000000000..49c7e72e48 --- /dev/null +++ b/qa/L0_perf_analyzer/test.sh @@ -0,0 +1,1164 @@ +#!/bin/bash +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "$REPO_VERSION" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi + +export CUDA_VISIBLE_DEVICES=0 + +CLIENT_LOG="./perf_analyzer.log" +PERF_ANALYZER=../clients/perf_analyzer + +DATADIR=`pwd`/models +TESTDATADIR=`pwd`/test_data + +INT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/int_data.json +INT_DIFFSHAPE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/int_data_diff_shape.json +INT_OPTIONAL_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/int_data_optional.json +FLOAT_DIFFSHAPE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/float_data_with_shape.json +STRING_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/string_data.json +STRING_WITHSHAPE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/string_data_with_shape.json +SEQ_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/seq_data.json +SHAPETENSORADTAFILE=`pwd`/../common/perf_analyzer_input_data_json/shape_tensor_data.json +IMAGE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/image_data.json + +OUTPUT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/output.json +NON_ALIGNED_OUTPUT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/non_aligned_output.json +WRONG_OUTPUT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/wrong_output.json +WRONG_OUTPUT_2_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/wrong_output_2.json + +SEQ_OUTPUT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/seq_output.json +SEQ_WRONG_OUTPUT_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/seq_wrong_output.json + +REPEAT_INT32_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/repeat_int32_data.json + +TRACE_FILE="trace.json" + +SERVER=/opt/tritonserver/bin/tritonserver +SERVER_ARGS="--model-repository=${DATADIR} --trace-config triton,file=${TRACE_FILE}" +SERVER_LOG="./inference_server.log" + +ERROR_STRING="error | Request count: 0 | : 0 infer/sec" + +STABILITY_THRESHOLD="100" + +source ../common/util.sh + +rm -f $SERVER_LOG $CLIENT_LOG +rm -rf $DATADIR $TESTDATADIR $ENSEMBLE_DATADIR + +mkdir -p $DATADIR +# Copy fixed-shape models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_int32_int32_int32 $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_nobatch_int32_int32_int32 $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_object_object_object $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_nobatch_object_object_object $DATADIR/ + +# Copy a variable-shape models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/graphdef_object_int32_int32 $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/graphdef_int32_int32_float32 $DATADIR/ + +# Copy shape tensor models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_shapetensor_model_repository/plan_zero_1_float32_int32 $DATADIR/ + +# Copying ensemble including a sequential model +cp -r /data/inferenceserver/${REPO_VERSION}/qa_sequence_model_repository/savedmodel_sequence_object $DATADIR +cp -r /data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_sequence_model_repository/simple_savedmodel_sequence_object $DATADIR +cp -r /data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_sequence_model_repository/nop_TYPE_FP32_-1 $DATADIR + +# Copying variable sequence model +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_sequence_model_repository/graphdef_sequence_float32 $DATADIR + +mkdir $DATADIR/nop_TYPE_FP32_-1/1 + +# Copy inception model to the model repository +cp -r /data/inferenceserver/${REPO_VERSION}/tf_model_store/inception_v1_graphdef $DATADIR + +# Copy resnet50v1.5_fp16 +cp -r /data/inferenceserver/${REPO_VERSION}/perf_model_store/resnet50v1.5_fp16_savedmodel $DATADIR + +# Copy and customize custom_zero_1_float32 +cp -r ../custom_models/custom_zero_1_float32 $DATADIR && \ + mkdir $DATADIR/custom_zero_1_float32/1 && \ + (cd $DATADIR/custom_zero_1_float32 && \ + echo "parameters [" >> config.pbtxt && \ + echo "{ key: \"execute_delay_ms\"; value: { string_value: \"100\" }}" >> config.pbtxt && \ + echo "]" >> config.pbtxt) + +# Copy and customize optional inputs model +cp -r ../python_models/optional $DATADIR && \ + mkdir $DATADIR/optional/1 && \ + mv $DATADIR/optional/model.py $DATADIR/optional/1 && \ + sed -i 's/max_batch_size: 0/max_batch_size: 2/g' $DATADIR/optional/config.pbtxt + +# Copy decoupled model +git clone --depth=1 https://github.com/triton-inference-server/python_backend +mkdir -p $DATADIR/repeat_int32/1 +cp python_backend/examples/decoupled/repeat_config.pbtxt $DATADIR/repeat_int32/config.pbtxt +cp python_backend/examples/decoupled/repeat_model.py $DATADIR/repeat_int32/1/model.py + +# Generating test data +mkdir -p $TESTDATADIR +for INPUT in INPUT0 INPUT1; do + for i in {1..16}; do + echo '1' >> $TESTDATADIR/${INPUT} + done +done + +RET=0 + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + + +# Test whether there was a conflict in sending sequences. This should +# be done before other testing as the server might emit this warning +# in certain test cases that are expected to raise this warning +SERVER_ERROR_STRING="The previous sequence did not end before this sequence start" + +set +e +$PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_object_object -p2000 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed: Expected an error when using dynamic shapes in string inputs\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "input INPUT0 contains dynamic shape, provide shapes to send along with the request" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed: \n***" + RET=1 +fi + +# Testing with ensemble and sequential model variants +$PERF_ANALYZER -v -i grpc -m simple_savedmodel_sequence_object -p 2000 -t5 --streaming \ +--input-data=$SEQ_JSONDATAFILE --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed: Sequence conflict when maintaining concurrency\n***" + RET=1 +fi + +$PERF_ANALYZER -v -i grpc -m simple_savedmodel_sequence_object -p 1000 --request-rate-range 100:200:50 --streaming \ +--input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +if [ $(cat $SERVER_LOG | grep "${SERVER_ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $SERVER_LOG | grep "${SERVER_ERROR_STRING}" + echo -e "\n***\n*** Test Failed: Sequence conflict\n***" + RET=1 +fi +set -e + +for PROTOCOL in grpc http; do + + # Testing simple configurations with different shared memory types + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 -t 1 -p2000 -b 1 \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 -t 1 -p2000 -b 1 -a \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # TODO Add back testing with preprocess_inception_ensemble model + + # Testing with inception model + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m inception_v1_graphdef -t 1 -p2000 -b 1 \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m inception_v1_graphdef -t 1 -p2000 -b 1 -a \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Testing with resnet50 models with large batch sizes + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m inception_v1_graphdef -t 2 -p2000 -b 64 \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m inception_v1_graphdef -t 2 -p2000 -b 64 \ + --shared-memory=$SHARED_MEMORY_TYPE -a -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Test perf client behavior on different model with different batch size + for MODEL in graphdef_nobatch_int32_int32_int32 graphdef_int32_int32_int32; do + # Valid batch size + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m $MODEL -t 1 -p2000 -b 1 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + # Invalid batch sizes + for STATIC_BATCH in 0 10; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m $MODEL -t 1 -p2000 -b $STATIC_BATCH -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + done + + # Testing with the new arguments + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 --concurrency-range 1:5:2 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "error | Request count: 0 | : 0 infer/sec\|: 0 usec|Request concurrency: 2" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 --concurrency-range 1:5:2 \ + --input-data=${INT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "error | Request count: 0 | : 0 infer/sec\|: 0 usec|Request concurrency: 2" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 --request-rate-range 1000:2000:500 \ + -p1000 -b 1 -a -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 --request-rate-range 1000:2000:500 \ + --input-data=${INT_JSONDATAFILE} -p1000 -b 1 -a -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + # Binary search for request rate mode + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_int32 --request-rate-range 1000:2000:100 -p1000 -b 1 \ + -a --binary-search --request-distribution "poisson" -l 10 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + # Binary search for concurrency range mode and make sure it doesn't hang + $PERF_ANALYZER -v -a --request-distribution "poisson" --shared-memory none \ + --percentile 99 --binary-search --concurrency-range 1:8:2 -l 5 \ + -m graphdef_int32_int32_int32 -b 1 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 & + PA_PID=$! + if [ "$PA_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $PERF_ANALYZER\n***" + cat $CLIENT_LOG + RET=1 + fi + # wait for PA to finish running + sleep 200 + if ps -p $PA_PID > /dev/null; then + cat $CLIENT_LOG + echo -e "\n***\n*** $PERF_ANALYZER is hanging after 200 s\n***" + kill $PA_PID + RET=1 + fi + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + # Testing with combinations of string input and shared memory types + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_object_object --string-data=1 -p2000 \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Testing with combinations of file inputs and shared memory types + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_object_object --input-data=$TESTDATADIR -p2000 \ + --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_object_object --input-data=$STRING_JSONDATAFILE \ + --input-data=$STRING_JSONDATAFILE -p2000 --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Testing with combinations of variable inputs and shared memory types + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_int32_int32 --input-data=$TESTDATADIR \ + --shape INPUT0:2,8 --shape INPUT1:2,8 -p2000 --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} \ + >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_object_int32_int32 --input-data=$STRING_WITHSHAPE_JSONDATAFILE \ + --shape INPUT0:2,8 --shape INPUT1:2,8 -p2000 --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} \ + >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_float32 --shape INPUT0:2,8,2 \ + --shape INPUT1:2,8,2 -p2000 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + # Trying to batch tensors with different shape + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m graphdef_int32_int32_float32 --shape INPUT0:2,8,2 --shape INPUT1:2,8,2 -p2000 -b 4 \ + --shared-memory=$SHARED_MEMORY_TYPE --input-data=$INT_DIFFSHAPE_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep -P "The supplied shape .+ is incompatible with the model's input shape" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Shape tensor I/O model (server needs the shape tensor on the CPU) + for SHARED_MEMORY_TYPE in none system; do + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m plan_zero_1_float32_int32 --input-data=$SHAPETENSORADTAFILE \ + --shape DUMMY_INPUT0:4,4 -p2000 --shared-memory=$SHARED_MEMORY_TYPE -b 8 -s ${STABILITY_THRESHOLD} \ + >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep ": 0 infer/sec\|: 0 usec" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 2000 -t5 --sync \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 2000 -t5 --sync \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + $PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 1000 --request-rate-range 100:200:50 --sync \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + + + # Testing with variable ensemble model. This unit specifies different shape values + # for different inferences. + for SHARED_MEMORY_TYPE in none system cuda; do + set +e + # FIXME: Enable HTTP when the server is able to correctly return the complex error messages. + $PERF_ANALYZER -v -i grpc -m graphdef_sequence_float32 --shape INPUT:2 --input-data=$FLOAT_DIFFSHAPE_JSONDATAFILE \ + --input-data=$FLOAT_DIFFSHAPE_JSONDATAFILE -p2000 --shared-memory=$SHARED_MEMORY_TYPE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep -P "The supplied shape .+ is incompatible with the model's input shape" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + set -e + done + + # Testing that trace logging works + set +e + rm ${TRACE_FILE}* + $PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 2000 -t5 --sync \ + --trace-level TIMESTAMPS --trace-rate 1000 --trace-count 100 --log-frequency 10 \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if ! compgen -G "$TRACE_FILE*" > /dev/null; then + echo -e "\n***\n*** Test Failed. $TRACE_FILE failed to generate.\n***" + RET=1 + elif [ $(cat ${TRACE_FILE}* | grep "REQUEST_START" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed. Did not find `REQUEST_START` in $TRACE_FILE \n***" + RET=1 + fi + curl localhost:8000/v2/trace/setting -d '{"trace_level":["OFF"]}' + set -e + + # Testing that setting trace file does not work + set +e + $PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object \ + --trace-file $TRACE_FILE >$CLIENT_LOG 2>&1 + if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed. Expected to fail for unknown arg --trace-file" + RET=1 + fi + curl localhost:8000/v2/trace/setting -d '{"trace_level":["OFF"]}' + set -e +done + +# Test with output validation +set +e +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 --input-data=${NON_ALIGNED_OUTPUT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "The 'validation_data' field doesn't align with 'data' field in the json file" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 --input-data=${WRONG_OUTPUT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "mismatch in the data provided" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 --input-data=${WRONG_OUTPUT_2_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "Output doesn't match expected output" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + + +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 --input-data=${OUTPUT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m simple_savedmodel_sequence_object -i grpc --streaming \ +--input-data=${SEQ_WRONG_OUTPUT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "Output doesn't match expected output" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m simple_savedmodel_sequence_object -i grpc --streaming \ +--input-data=${SEQ_OUTPUT_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +## Testing with very large concurrencies and large dataset +INPUT_DATA_OPTION="--input-data $SEQ_JSONDATAFILE " +for i in {1..9}; do + INPUT_DATA_OPTION=" ${INPUT_DATA_OPTION} ${INPUT_DATA_OPTION}" +done +set +e +$PERF_ANALYZER -v -m simple_savedmodel_sequence_object -p 10000 --concurrency-range 1500:2000:250 -i grpc --streaming \ +${INPUT_DATA_OPTION} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +## Test count_windows mode +set +e + +# Send incorrect shape and make sure that perf_analyzer doesn't hang +$PERF_ANALYZER -v -m graphdef_object_int32_int32 --measurement-mode "count_windows" \ + --shape INPUT0:1,8,100 --shape INPUT1:2,8 --string-data=1 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "unexpected shape for input 'INPUT0' for model" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_object_int32_int32 --measurement-mode "count_windows" \ + --shape INPUT0:2,8 --shape INPUT1:2,8 --string-data=1 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +# Test with optional inputs missing but still valid +set +e +$PERF_ANALYZER -v -m optional --measurement-mode "count_windows" \ + --input-data=${INT_OPTIONAL_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +# Test with optional inputs missing and invalid +set +e +OPTIONAL_INPUT_ERROR_STRING="For batch sizes larger than 1, the same set of +inputs must be specified for each batch. You cannot use different set of +optional inputs for each individual batch." +$PERF_ANALYZER -v -m optional -b 2 --measurement-mode "count_windows" \ + --input-data=${INT_OPTIONAL_JSONDATAFILE} -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${OPTIONAL_INPUT_ERROR_STRING}" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + + +# Test Custom request rate option +CUSTOM_SCHEDULE_FILE=$TESTDATADIR/custom.schedule +echo '30000' >> $CUSTOM_SCHEDULE_FILE +echo '10000' >> $CUSTOM_SCHEDULE_FILE +echo '40000' >> $CUSTOM_SCHEDULE_FILE +echo '20000' >> $CUSTOM_SCHEDULE_FILE +echo '25000' >> $CUSTOM_SCHEDULE_FILE + +set +e +$PERF_ANALYZER -v -i grpc -m graphdef_int32_int32_int32 --request-intervals $CUSTOM_SCHEDULE_FILE >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "Request Rate: 40" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed: \n***" + RET=1 +fi +set -e + +# Test --serial-sequences mode +set +e +$PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 1000 --request-rate-range 100:200:50 --serial-sequences \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -i $PROTOCOL -m simple_savedmodel_sequence_object -p 1000 --request-intervals $CUSTOM_SCHEDULE_FILE --serial-sequences \ + --input-data=$SEQ_JSONDATAFILE -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +## Test decoupled model support +$PERF_ANALYZER -v -m repeat_int32 --input-data=$REPEAT_INT32_JSONDATAFILE \ + --profile-export-file profile_export.json -i grpc --async --streaming -s \ + ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +python3 -c "import json ; \ + requests = json.load(open('profile_export.json'))['experiments'][0]['requests'] ; \ + assert any(len(r['response_timestamps']) > 1 for r in requests)" +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +check-jsonschema --schemafile perf_analyzer_profile_export_schema.json profile_export.json +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +## Test perf_analyzer with MPI / multiple models + +is_synchronized() { + local TIMESTAMP_RANK_0_STABLE=$(grep -oP "^\K[^$]+(?=\[1,0\]:All models on all MPI ranks are stable)" 1/rank.0/stdout | date "+%s" -f -) + local TIMESTAMP_RANK_1_STABLE=$(grep -oP "^\K[^$]+(?=\[1,1\]:All models on all MPI ranks are stable)" 1/rank.1/stdout | date "+%s" -f -) + local TIMESTAMP_RANK_2_STABLE=$(grep -oP "^\K[^$]+(?=\[1,2\]:All models on all MPI ranks are stable)" 1/rank.2/stdout | date "+%s" -f -) + local TIMESTAMP_MIN=$(echo -e "${TIMESTAMP_RANK_0_STABLE}\n${TIMESTAMP_RANK_1_STABLE}\n${TIMESTAMP_RANK_2_STABLE}" | sort -n | head -1) + local TIMESTAMP_MAX=$(echo -e "${TIMESTAMP_RANK_0_STABLE}\n${TIMESTAMP_RANK_1_STABLE}\n${TIMESTAMP_RANK_2_STABLE}" | sort -n | tail -1) + local TIMESTAMP_MAX_MIN_DIFFERENCE=$((${TIMESTAMP_MAX}-${TIMESTAMP_MIN})) + local ALLOWABLE_SECONDS_BETWEEN_PROFILES_FINISHING="5" + echo $(($TIMESTAMP_MAX_MIN_DIFFERENCE <= $ALLOWABLE_SECONDS_BETWEEN_PROFILES_FINISHING)) +} + +is_stable() { + local RANK=$1 + local IS_THROUGHPUT=$2 + if [ $IS_THROUGHPUT ]; then + local GREP_PATTERN="\[1,$RANK\]: Pass \[[0-9]+\] throughput: \K[0-9]+\.?[0-9]*" + else + local GREP_PATTERN="\[1,$RANK\]: Pass \[[0-9]+\] throughput: [0-9]+\.?[0-9]* infer/sec. Avg latency: \K[0-9]+" + fi + local LAST_MINUS_0=$(grep -oP "$GREP_PATTERN" 1/rank.$RANK/stdout | tail -3 | sed -n 3p) + local LAST_MINUS_1=$(grep -oP "$GREP_PATTERN" 1/rank.$RANK/stdout | tail -3 | sed -n 2p) + local LAST_MINUS_2=$(grep -oP "$GREP_PATTERN" 1/rank.$RANK/stdout | tail -3 | sed -n 1p) + local MEAN=$(awk "BEGIN {print (($LAST_MINUS_0+$LAST_MINUS_1+$LAST_MINUS_2)/3)}") + local STABILITY_THRESHOLD=0.5 + # Based on this: https://github.com/triton-inference-server/client/blob/main/src/c++/perf_analyzer/inference_profiler.cc#L629-L644 + local WITHIN_THRESHOLD_0=$(awk "BEGIN {print ($LAST_MINUS_0 >= ((1 - $STABILITY_THRESHOLD) * $MEAN) && $LAST_MINUS_0 <= ((1 + $STABILITY_THRESHOLD) * $MEAN))}") + local WITHIN_THRESHOLD_1=$(awk "BEGIN {print ($LAST_MINUS_1 >= ((1 - $STABILITY_THRESHOLD) * $MEAN) && $LAST_MINUS_1 <= ((1 + $STABILITY_THRESHOLD) * $MEAN))}") + local WITHIN_THRESHOLD_2=$(awk "BEGIN {print ($LAST_MINUS_2 >= ((1 - $STABILITY_THRESHOLD) * $MEAN) && $LAST_MINUS_2 <= ((1 + $STABILITY_THRESHOLD) * $MEAN))}") + echo $(($WITHIN_THRESHOLD_0 && $WITHIN_THRESHOLD_1 && $WITHIN_THRESHOLD_2)) +} + +set +e +mpiexec --allow-run-as-root \ + -n 1 --merge-stderr-to-stdout --output-filename . --tag-output --timestamp-output \ + $PERF_ANALYZER -v -m graphdef_int32_int32_int32 \ + --measurement-mode count_windows -s 50 --enable-mpi : \ + -n 1 --merge-stderr-to-stdout --output-filename . --tag-output --timestamp-output \ + $PERF_ANALYZER -v -m graphdef_nobatch_int32_int32_int32 \ + --measurement-mode count_windows -s 50 --enable-mpi : \ + -n 1 --merge-stderr-to-stdout --output-filename . --tag-output --timestamp-output \ + $PERF_ANALYZER -v -m custom_zero_1_float32 \ + --measurement-mode count_windows -s 50 --enable-mpi +if [ $? -ne 0 ]; then + cat 1/rank.0/stdout 1/rank.2/stdout 1/rank.2/stdout + echo -e "\n***\n*** Perf Analyzer returned non-zero exit code\n***" + echo -e "\n***\n*** Test Failed\n***" + RET=1 +else + if [ $(is_synchronized) -eq 0 ]; then + cat 1/rank.0/stdout 1/rank.2/stdout 1/rank.2/stdout + echo -e "\n***\n*** All models did not finish profiling at almost the same time\n***" + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + RANK_0_THROUGHPUT_IS_STABLE=$(is_stable 0 1) + RANK_0_LATENCY_IS_STABLE=$(is_stable 0 0) + RANK_1_THROUGHPUT_IS_STABLE=$(is_stable 1 1) + RANK_1_LATENCY_IS_STABLE=$(is_stable 1 0) + RANK_2_THROUGHPUT_IS_STABLE=$(is_stable 2 1) + RANK_2_LATENCY_IS_STABLE=$(is_stable 2 0) + + ALL_STABLE=$(( \ + $RANK_0_THROUGHPUT_IS_STABLE && \ + $RANK_0_LATENCY_IS_STABLE && \ + $RANK_1_THROUGHPUT_IS_STABLE && \ + $RANK_1_LATENCY_IS_STABLE && \ + $RANK_2_THROUGHPUT_IS_STABLE && \ + $RANK_2_LATENCY_IS_STABLE)) + + if [ $ALL_STABLE -eq 0 ]; then + cat 1/rank.0/stdout 1/rank.2/stdout 1/rank.2/stdout + echo -e "\n***\n*** All models did not stabilize\n***" + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + rm -rf 1 +fi +set -e + +## Test perf_analyzer without MPI library (`libmpi.so`) available + +rm -rf /opt/hpcx/ompi/lib/libmpi* + +set +e +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 -s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +kill $SERVER_PID +wait $SERVER_PID + +# Generate valid CA +openssl genrsa -passout pass:1234 -des3 -out ca.key 4096 +openssl req -passin pass:1234 -new -x509 -days 365 -key ca.key -out ca.crt -subj "/C=SP/ST=Spain/L=Valdepenias/O=Test/OU=Test/CN=Root CA" + +# Generate valid Server Key/Cert +openssl genrsa -passout pass:1234 -des3 -out server.key 4096 +openssl req -passin pass:1234 -new -key server.key -out server.csr -subj "/C=SP/ST=Spain/L=Valdepenias/O=Test/OU=Server/CN=localhost" +openssl x509 -req -passin pass:1234 -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt + +# Remove passphrase from the Server Key +openssl rsa -passin pass:1234 -in server.key -out server.key + +# Generate valid Client Key/Cert +openssl genrsa -passout pass:1234 -des3 -out client.key 4096 +openssl req -passin pass:1234 -new -key client.key -out client.csr -subj "/C=SP/ST=Spain/L=Valdepenias/O=Test/OU=Client/CN=localhost" +openssl x509 -passin pass:1234 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out client.crt + +# Remove passphrase from Client Key +openssl rsa -passin pass:1234 -in client.key -out client.key + +# Create mutated client key (Make first char of each like capital) +cp client.key client2.key && sed -i "s/\b\(.\)/\u\1/g" client2.key +cp client.crt client2.crt && sed -i "s/\b\(.\)/\u\1/g" client2.crt + +SERVER_ARGS="--model-repository=${DATADIR} --grpc-use-ssl=1 --grpc-server-cert=server.crt --grpc-server-key=server.key --grpc-root-cert=ca.crt" + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +# Test gRPC SSL +set +e + +# Test that gRPC protocol with SSL works correctly +$PERF_ANALYZER -v -i grpc -m graphdef_int32_int32_int32 \ + --ssl-grpc-use-ssl \ + --ssl-grpc-root-certifications-file=ca.crt \ + --ssl-grpc-private-key-file=client.key \ + --ssl-grpc-certificate-chain-file=client.crt \ + -s ${STABILITY_THRESHOLD} \ + > ${CLIENT_LOG}.grpc_success 2>&1 +if [ $? -ne 0 ]; then + cat ${CLIENT_LOG}.grpc_success + RET=1 +fi + +# Test that gRPC protocol with SSL fails with incorrect key +$PERF_ANALYZER -v -i grpc -m graphdef_int32_int32_int32 \ + --ssl-grpc-use-ssl \ + --ssl-grpc-root-certifications-file=ca.crt \ + --ssl-grpc-private-key-file=client.key \ + --ssl-grpc-certificate-chain-file=client2.crt \ + -s ${STABILITY_THRESHOLD} \ + > ${CLIENT_LOG}.grpc_failure 2>&1 +if [ $? -eq 0 ]; then + cat ${CLIENT_LOG}.grpc_failure + echo -e "\n***\n*** Expected test failure\n***" + RET=1 +fi + +set -e + +kill $SERVER_PID +wait $SERVER_PID + +cp server.crt /etc/nginx/cert.crt +cp server.key /etc/nginx/cert.key + +SERVER_ARGS="--model-repository=${DATADIR}" + +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +# Setup the new configuration for the proxy. The HTTPS traffic will be +# redirected to the running instance of server at localhost:8000 +cp nginx.conf /etc/nginx/sites-available/default + +# Start the proxy server +service nginx restart + +# Test HTTP SSL +set +e + +# Test that HTTP protocol with SSL works correctly with certificates +$PERF_ANALYZER -v -u https://localhost:443 -i http -m graphdef_int32_int32_int32 \ + --ssl-https-verify-peer 1 \ + --ssl-https-verify-host 2 \ + --ssl-https-ca-certificates-file ca.crt \ + --ssl-https-client-certificate-file client.crt \ + --ssl-https-client-certificate-type PEM \ + --ssl-https-private-key-file client.key \ + --ssl-https-private-key-type PEM \ + -s ${STABILITY_THRESHOLD} \ + > ${CLIENT_LOG}.https_success 2>&1 +if [ $? -ne 0 ]; then + cat ${CLIENT_LOG}.https_success + RET=1 +fi + +# Test that HTTP protocol with SSL works correctly without certificates +$PERF_ANALYZER -v -u https://localhost:443 -i http -m graphdef_int32_int32_int32 \ + --ssl-https-verify-peer 0 \ + --ssl-https-verify-host 0 \ + -s ${STABILITY_THRESHOLD} \ + > ${CLIENT_LOG}.https_success 2>&1 +if [ $? -ne 0 ]; then + cat ${CLIENT_LOG}.https_success + RET=1 +fi + +# Test that HTTP protocol with SSL fails with incorrect key +$PERF_ANALYZER -v -u https://localhost:443 -i http -m graphdef_int32_int32_int32 \ + --ssl-https-verify-peer 1 \ + --ssl-https-verify-host 2 \ + --ssl-https-ca-certificates-file ca.crt \ + --ssl-https-client-certificate-file client.crt \ + --ssl-https-client-certificate-type PEM \ + --ssl-https-private-key-file client2.key \ + --ssl-https-private-key-type PEM \ + -s ${STABILITY_THRESHOLD} \ + > ${CLIENT_LOG}.https_failure 2>&1 +if [ $? -eq 0 ]; then + cat ${CLIENT_LOG}.https_failure + echo -e "\n***\n*** Expected test failure\n***" + RET=1 +fi + +set -e + +kill $SERVER_PID +wait $SERVER_PID + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit $RET diff --git a/qa/L0_perf_analyzer_capi/test.sh b/qa/L0_perf_analyzer_capi/test.sh new file mode 100755 index 0000000000..53196fa762 --- /dev/null +++ b/qa/L0_perf_analyzer_capi/test.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# TESTS COPIED FROM L0_perf_analyzer/test.sh +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "$REPO_VERSION" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi + +export CUDA_VISIBLE_DEVICES=0 + +CLIENT_LOG="./perf_analyzer.log" +PERF_ANALYZER=../clients/perf_analyzer + +DATADIR=`pwd`/models +TESTDATADIR=`pwd`/test_data + +SERVER_LIBRARY_PATH=/opt/tritonserver + +FLOAT_DIFFSHAPE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/float_data_with_shape.json +STRING_WITHSHAPE_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/string_data_with_shape.json +SEQ_JSONDATAFILE=`pwd`/../common/perf_analyzer_input_data_json/seq_data.json +SHAPETENSORADTAFILE=`pwd`/../common/perf_analyzer_input_data_json/shape_tensor_data.json + +ERROR_STRING="error | Request count: 0 | : 0 infer/sec" + +STABILITY_THRESHOLD="9999" + +source ../common/util.sh + +rm -f $CLIENT_LOG +rm -rf $DATADIR $TESTDATADIR $ENSEMBLE_DATADIR + +mkdir -p $DATADIR +# Copy fixed-shape models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_int32_int32_int32 $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_object_object_object $DATADIR/ + +# Copy a variable-shape models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/graphdef_object_int32_int32 $DATADIR/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_model_repository/graphdef_int32_int32_float32 $DATADIR/ + +# Copy shape tensor models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_shapetensor_model_repository/plan_zero_1_float32_int32 $DATADIR/ + +# Copying ensemble including a sequential model +cp -r /data/inferenceserver/${REPO_VERSION}/qa_sequence_model_repository/savedmodel_sequence_object $DATADIR +cp -r /data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_sequence_model_repository/simple_savedmodel_sequence_object $DATADIR + +# Copying variable sequence model +cp -r /data/inferenceserver/${REPO_VERSION}/qa_variable_sequence_model_repository/graphdef_sequence_float32 $DATADIR + +# Copying bls model with undefined variable +mkdir -p $DATADIR/bls_undefined/1 && \ + cp ../python_models/bls_undefined/model.py $DATADIR/bls_undefined/1/. && \ + cp ../python_models/bls_undefined/config.pbtxt $DATADIR/bls_undefined/. + +# Generating test data +mkdir -p $TESTDATADIR +for INPUT in INPUT0 INPUT1; do + for i in {1..16}; do + echo '1' >> $TESTDATADIR/${INPUT} + done +done + +RET=0 + +########## Test C API ############# +# Make sure tritonserver is not running first +set +e +SERVER_PID=$(pidof tritonserver) +if [ $? -ne 1 ]; then +echo -e "\n There was a previous instance of tritonserver, killing \n" + kill $SERVER_PID + wait $SERVER_PID +fi +set -e + +# Testing simple configuration +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 \ +--service-kind=triton_c_api \ +--model-repository=$DATADIR --triton-server-directory=$SERVER_LIBRARY_PATH \ +-s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 -t 1 -p2000 -b 1 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +#Testing with string input +$PERF_ANALYZER -v -m graphdef_object_object_object --string-data=1 -p2000 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +# Testing with variable inputs +$PERF_ANALYZER -v -m graphdef_object_int32_int32 --input-data=$TESTDATADIR \ +--shape INPUT0:2,8 --shape INPUT1:2,8 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_object_int32_int32 \ +--input-data=$STRING_WITHSHAPE_JSONDATAFILE \ +--shape INPUT0:2,8 --shape INPUT1:2,8 -p2000 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m graphdef_int32_int32_float32 --shape INPUT0:2,8,2 \ +--shape INPUT1:2,8,2 -p2000 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +# Shape tensor I/O model (server needs the shape tensor on the CPU) +$PERF_ANALYZER -v -m plan_zero_1_float32_int32 --input-data=$SHAPETENSORADTAFILE \ +--shape DUMMY_INPUT0:4,4 -p2000 -b 8 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep ": 0 infer/sec\|: 0 usec" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +$PERF_ANALYZER -v -m simple_savedmodel_sequence_object -p 2000 -t5 --sync \ +-s ${STABILITY_THRESHOLD} \ +--input-data=$SEQ_JSONDATAFILE \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH >$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi + +set +e +$PERF_ANALYZER -v -m graphdef_sequence_float32 --shape INPUT:2 \ +-s ${STABILITY_THRESHOLD} \ +--input-data=$FLOAT_DIFFSHAPE_JSONDATAFILE \ +--input-data=$FLOAT_DIFFSHAPE_JSONDATAFILE -p2000 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH --sync >$CLIENT_LOG 2>&1 +if [ $? -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep -P "The supplied shape .+ is incompatible with the model's input shape" | wc -l) -eq 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +for SHARED_MEMORY_TYPE in system cuda; do + $PERF_ANALYZER -v -m graphdef_int32_int32_int32 -t 1 -p2000 -b 1 \ + -s ${STABILITY_THRESHOLD} \ + --shared-memory=$SHARED_MEMORY_TYPE \ + --service-kind=triton_c_api --model-repository=$DATADIR \ + --triton-server-directory=$SERVER_LIBRARY_PATH >$CLIENT_LOG 2>&1 + if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi +done + + +$PERF_ANALYZER -v -m graphdef_int32_int32_int32 --request-rate-range 1000:2000:500 -p1000 -b 1 \ +--service-kind=triton_c_api --model-repository=$DATADIR \ +--triton-server-directory=$SERVER_LIBRARY_PATH -s ${STABILITY_THRESHOLD} \ +>$CLIENT_LOG 2>&1 +if [ $? -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +set +e +# Testing erroneous configuration +# This model is expected to fail +$PERF_ANALYZER -v -m bls_undefined --shape INPUT0:1048576 -t 64\ +--service-kind=triton_c_api \ +--model-repository=$DATADIR --triton-server-directory=$SERVER_LIBRARY_PATH \ +-s ${STABILITY_THRESHOLD} >$CLIENT_LOG 2>&1 +if [ $? -ne 99 ]; then + cat $CLIENT_LOG + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e + +# Make sure server is not still running +set +e +SERVER_PID=$(pidof tritonserver) +if [ $? -eq 0 ]; then + echo -e "\n Tritonserver did not exit properly, killing \n" + kill $SERVER_PID + wait $SERVER_PID + RET=1 +fi +set -e + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo -e "\n***\n*** Test FAILED\n***" +fi +exit $RET diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt b/qa/L0_perf_analyzer_doc_links/mkdocs.yml similarity index 85% rename from python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt rename to qa/L0_perf_analyzer_doc_links/mkdocs.yml index 39b3c48edb..41a4bfe485 100644 --- a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt +++ b/qa/L0_perf_analyzer_doc_links/mkdocs.yml @@ -1,4 +1,4 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,5 +24,13 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -backend: "vllm" -instance_group [{kind: KIND_MODEL}] +site_name: CI Test +use_directory_urls: False +docs_dir: "./docs" +plugins: + - htmlproofer + - search + +markdown_extensions: + - toc: + permalink: True diff --git a/qa/L0_backend_onnxruntime/test.sh b/qa/L0_perf_analyzer_doc_links/test.sh similarity index 55% rename from qa/L0_backend_onnxruntime/test.sh rename to qa/L0_perf_analyzer_doc_links/test.sh index 2e66b467e9..d0757bca9e 100755 --- a/qa/L0_backend_onnxruntime/test.sh +++ b/qa/L0_perf_analyzer_doc_links/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,67 +25,50 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -DATADIR=/data/inferenceserver/${REPO_VERSION} -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_LOG="./inference_server.log" -CLIENT_LOG="./test.log" -source ../common/util.sh - -rm -f *.log -rm -rf models - +LOG="`pwd`/doc_links.log" +CONFIG="`pwd`/mkdocs.yml" RET=0 -# BFLOAT16 test -mkdir -p models -cp -r ${DATADIR}/qa_model_repository/onnx_bf16_bf16_bf16 models/. +# Download necessary packages +python3 -m pip install mkdocs +python3 -m pip install mkdocs-htmlproofer-plugin==0.10.3 -SERVER_ARGS="--model-repository=`pwd`/models" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi +#Download perf_analyzer docs +TRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION:="http://github.com/triton-inference-server"} +TRITON_PERF_ANALYZER_REPO_TAG="${TRITON_PERF_ANALYZER_REPO_TAG:=main}" +git clone -b ${TRITON_PERF_ANALYZER_REPO_TAG} ${TRITON_REPO_ORGANIZATION}/perf_analyzer.git +cp `pwd`/perf_analyzer/README.md . +cp -rf `pwd`/perf_analyzer/docs . + +# Need to remove all links that start with -- or -. Mkdocs converts all -- to - for anchor links. +# This breaks all links to cli commands throughout the docs. This will iterate over all +# files in the docs directory and remove -- and - at the start of options, which allows the +# tool to check links for correctness. +for file in `pwd`/docs/*.md +do + echo $file + sed -i 's/`-*/`/g' $file + sed -i 's/#-*/#/g' $file +done -set +e +exec mkdocs serve -f $CONFIG > $LOG & +PID=$! +sleep 20 -for client_type in http grpc; do - export CLIENT_TYPE=$client_type - CLIENT_LOG="./test_${client_type}.log" - python test.py >>$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed ($client_type)\n***" - RET=1 - fi +until [[ (-z `pgrep mkdocs`) ]]; do + kill -2 $PID + sleep 2 done -unset CLIENT_TYPE -set -e +if [[ ! -z `grep "invalid url" $LOG` ]]; then + cat $LOG + RET=1 +fi -kill $SERVER_PID -wait $SERVER_PID if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" + echo -e "\n***\n*** Test PASSED\n***" else echo -e "\n***\n*** Test FAILED\n***" fi - exit $RET diff --git a/qa/L0_perf_analyzer_ground_truth/test.sh b/qa/L0_perf_analyzer_ground_truth/test.sh new file mode 100755 index 0000000000..d5d78e63f4 --- /dev/null +++ b/qa/L0_perf_analyzer_ground_truth/test.sh @@ -0,0 +1,175 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "${REPO_VERSION}" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi + +source ../common/util.sh + +# Setup client/perf_analyzer +CLIENT_LOG="./perf_analyzer.log" +PERF_ANALYZER=../clients/perf_analyzer + +function check_perf_analyzer_error { + ERROR_STRING="error | Request count: 0 | : 0 infer/sec" + CLIENT_RET="$1" + if [ ${CLIENT_RET} -ne 0 ]; then + cat ${CLIENT_LOG} + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat ${CLIENT_LOG} | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat ${CLIENT_LOG} + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi +} + +# Checks that the model infer/sec performance is equal to an expected value +# +/- some tolerance. +# $1: csv result file from PA run +# $2: expected infer/sec value +# $3: tolerance for expected value equality +function check_performance { + # get the boundary values based on the tolerance percentage + MIN=$(python3 -c "print(${2} * (1 - ${3}))") + MAX=$(python3 -c "print(${2} * (1 + ${3}))") + + # delete all but the 2nd line in the resulting file + # then get the 2nd column value which is the infer/sec measurement + report_val=$(sed '2!d' $1 | awk -F ',' {'print $2'}) + + # check if within tolerance + ret=$(python3 -c "print(${report_val} >= ${MIN} and ${report_val} <= ${MAX})") + if [ "$ret" = "False" ]; then + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi +} + +# Iterate over the grpc results to ensure gRPC times are greater than 0 +# $1: client log file +# example line: Avg gRPC time: 42648 usec (marshal 6 usec + response wait 42640 usec + unmarshal 2 usec) +function check_grpc_time { + grep "gRPC" $1 | awk '{print $4}' | while read -r line; do + if [ $line -eq 0 ]; then + RET=1 + fi + done +} + +# Create input_data.json to communicate the requested model delay +# $1: desired model delay +function create_input_data { + echo "{\"data\":[{\"INPUT0\" : [${1}]}]}" > input_data.json +} + +# Setup server +export CUDA_VISIBLE_DEVICES=0 +SERVER=/opt/tritonserver/bin/tritonserver +SERVER_ARGS="--model-repository=`pwd`/models" +SERVER_LOG="./inference_server.log" + +rm -f $SERVER_LOG $CLIENT_LOG +MODEL_DIR="./models" +rm -fr ${MODEL_DIR} && mkdir ${MODEL_DIR} +MODELS="ground_truth" + +for model in ${MODELS}; do + # Add version directory to each model if non-existent + mkdir -p "${MODEL_DIR}/${model}/1" + cp ../python_models/${model}/model.py ./models/${model}/1/model.py + cp ../python_models/${model}/config.pbtxt ./models/${model}/config.pbtxt +done + +# Run server +run_server +if [ "${SERVER_PID}" == "0" ]; then + echo -e "\n***\n*** Failed to start ${SERVER}\n***" + cat ${SERVER_LOG} + exit 1 +fi + +# Run perf_analyzer +set +e +RET=0 +PROTOCOLS="http grpc" +OUTPUT_FILE="results" +MODEL_DELAYS=(0.05 0.5) +TOLERANCE="0.05" + +for model_delay in ${MODEL_DELAYS[@]}; do + create_input_data ${model_delay} + EXPECTED_RESULT=$(python3 -c "print(1 / ${model_delay})") + for protocol in ${PROTOCOLS}; do + for model in ${MODELS}; do + echo "================================================================" + echo "[PERMUTATION] Protocol=${protocol} Model=${model}" + echo "================================================================" + + ${PERF_ANALYZER} -v -i ${protocol} --concurrency-range 2 --input-data input_data.json -m ${model} -f ${OUTPUT_FILE} | tee ${CLIENT_LOG} 2>&1 + check_perf_analyzer_error $? + + check_performance ${OUTPUT_FILE} ${EXPECTED_RESULT} ${TOLERANCE} + + if [ "${protocol}" == "grpc" ]; then + check_grpc_time ${CLIENT_LOG} + fi + done; + done; +done; + + +set -e + +# Cleanup +kill $SERVER_PID +wait $SERVER_PID + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo "=== START SERVER LOG ===" + cat ${SERVER_LOG} + echo "=== END SERVER LOG ===" + echo "=== START CLIENT LOG ===" + cat ${CLIENT_LOG} + echo "=== END CLIENT LOG ===" + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit ${RET} diff --git a/qa/L0_perf_analyzer_report/test.sh b/qa/L0_perf_analyzer_report/test.sh new file mode 100755 index 0000000000..469d11ce3a --- /dev/null +++ b/qa/L0_perf_analyzer_report/test.sh @@ -0,0 +1,175 @@ +#!/bin/bash +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} +if [ "$#" -ge 1 ]; then + REPO_VERSION=$1 +fi +if [ -z "${REPO_VERSION}" ]; then + echo -e "Repository version must be specified" + echo -e "\n***\n*** Test Failed\n***" + exit 1 +fi +if [ ! -z "$TEST_REPO_ARCH" ]; then + REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} +fi + +source ../common/util.sh + +# Setup client/perf_analyzer +CLIENT_LOG="./perf_analyzer.log" +PERF_ANALYZER=../clients/perf_analyzer + +function check_perf_analyzer_error { + ERROR_STRING="error | Request count: 0 | : 0 infer/sec" + CLIENT_RET="$1" + if [ ${CLIENT_RET} -ne 0 ]; then + cat ${CLIENT_LOG} + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + if [ $(cat ${CLIENT_LOG} | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then + cat ${CLIENT_LOG} + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi +} + +function check_cache_output { + # Validate cache info in perf_analyzer output + CACHE_STRING="Cache hit count" + if [ $(cat ${CLIENT_LOG} | grep -i "${CACHE_STRING}" | wc -l) -eq 0 ]; then + cat ${CLIENT_LOG} + echo "ERROR: No cache hit count found in output" + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi + + # Validate non-zero number of cache hits + ERROR_STRING="Cache hit count: 0" + num_cache_hit_lines=$(cat ${CLIENT_LOG} | grep -i "${CACHE_STRING}" | wc -l) + num_cache_hit_zero_lines=$(cat ${CLIENT_LOG} | grep -i "${ERROR_STRING}" | wc -l) + if [ ${num_cache_hit_zero_lines} -eq ${num_cache_hit_lines} ]; then + cat ${CLIENT_LOG} + echo "ERROR: All cache hit counts were zero, expected a non-zero number of cache hits" + echo -e "\n***\n*** Test Failed\n***" + RET=1 + fi +} + +# Setup server +export CUDA_VISIBLE_DEVICES=0 +SERVER=/opt/tritonserver/bin/tritonserver +# --response-cache-byte-size must be non-zero to test models with cache enabled +SERVER_ARGS="--model-repository=`pwd`/models --response-cache-byte-size=8192" +SERVER_LOG="./inference_server.log" + +# Setup model repository from existing qa_model_repository +rm -f $SERVER_LOG $CLIENT_LOG +MODEL_DIR="./models" +rm -fr ${MODEL_DIR} && mkdir ${MODEL_DIR} +ENSEMBLE_MODEL="simple_onnx_float32_float32_float32" +COMPOSING_MODEL="onnx_float32_float32_float32" +ENSEMBLE_MODEL_CACHE_ENABLED="${ENSEMBLE_MODEL}_cache_enabled" +ENSEMBLE_MODEL_CACHE_DISABLED="${ENSEMBLE_MODEL}_cache_disabled" +COMPOSING_MODEL_CACHE_ENABLED="${COMPOSING_MODEL}_cache_enabled" +COMPOSING_MODEL_CACHE_DISABLED="${COMPOSING_MODEL}_cache_disabled" +MODELS="${ENSEMBLE_MODEL_CACHE_ENABLED} ${ENSEMBLE_MODEL_CACHE_DISABLED} ${COMPOSING_MODEL_CACHE_ENABLED} ${COMPOSING_MODEL_CACHE_DISABLED}" + +## Setup ensemble models, one with cache enabled and one with cache disabled +cp -r "/data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_model_repository/${ENSEMBLE_MODEL}" "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_ENABLED}" +cp -r "/data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_model_repository/${ENSEMBLE_MODEL}" "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_DISABLED}" + +## Setup composing models, one with cache enabled and one with cache disabled +cp -r "/data/inferenceserver/${REPO_VERSION}/qa_model_repository/${COMPOSING_MODEL}" "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_ENABLED}" +cp -r "/data/inferenceserver/${REPO_VERSION}/qa_model_repository/${COMPOSING_MODEL}" "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_DISABLED}" + +for model in ${MODELS}; do + # Remove "name" line from each config to use directory name for simplicity + sed -i "/^name:/d" "${MODEL_DIR}/${model}/config.pbtxt" + # Add version directory to each model if non-existent + mkdir -p "${MODEL_DIR}/${model}/1" +done + +## Update "model_name" lines in each ensemble model config ensemble steps +sed -i "s/${COMPOSING_MODEL}/${COMPOSING_MODEL_CACHE_ENABLED}/g" "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_ENABLED}/config.pbtxt" +sed -i "s/${COMPOSING_MODEL}/${COMPOSING_MODEL_CACHE_DISABLED}/g" "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_DISABLED}/config.pbtxt" + +## Append cache config to each model config +echo -e "response_cache { enable: True }" >> "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_ENABLED}/config.pbtxt" +echo -e "response_cache { enable: False }" >> "${MODEL_DIR}/${ENSEMBLE_MODEL_CACHE_DISABLED}/config.pbtxt" +echo -e "response_cache { enable: True }" >> "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_ENABLED}/config.pbtxt" +echo -e "response_cache { enable: False }" >> "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_DISABLED}/config.pbtxt" +# Force CPU memory for composing models since cache doesn't currently support GPU memory +echo -e "instance_group [{ kind: KIND_CPU, count: 1 }]" >> "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_ENABLED}/config.pbtxt" +echo -e "instance_group [{ kind: KIND_CPU, count: 1 }]" >> "${MODEL_DIR}/${COMPOSING_MODEL_CACHE_DISABLED}/config.pbtxt" + +# Run server +run_server +if [ "${SERVER_PID}" == "0" ]; then + echo -e "\n***\n*** Failed to start ${SERVER}\n***" + cat ${SERVER_LOG} + exit 1 +fi + +# Run perf_analyzer +set +e +RET=0 +PROTOCOLS="http grpc" +STABILITY_THRESHOLD="15" +for protocol in ${PROTOCOLS}; do + for model in ${MODELS}; do + echo "================================================================" + echo "[PERMUTATION] Protocol=${protocol} Model=${model}" + echo "================================================================" + + ${PERF_ANALYZER} -v -i ${protocol} -m ${model} -s ${STABILITY_THRESHOLD} | tee ${CLIENT_LOG} 2>&1 + check_perf_analyzer_error $? + + # Check response cache outputs + if [[ ${model} == *"cache_enabled"* ]]; then + check_cache_output + fi + done; +done; +set -e + +# Cleanup +kill $SERVER_PID +wait $SERVER_PID + + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + echo "=== START SERVER LOG ===" + cat ${SERVER_LOG} + echo "=== END SERVER LOG ===" + echo -e "\n***\n*** Test FAILED\n***" +fi + +exit ${RET} diff --git a/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/config.pbtxt b/qa/L0_perf_analyzer_unit_tests/test.sh old mode 100644 new mode 100755 similarity index 75% rename from qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/config.pbtxt rename to qa/L0_perf_analyzer_unit_tests/test.sh index 1537b804ab..f2a70d23ff --- a/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/config.pbtxt +++ b/qa/L0_perf_analyzer_unit_tests/test.sh @@ -1,4 +1,5 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +#!/bin/bash +# Copyright 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,34 +25,26 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +TEST_LOG="./perf_analyzer_unit_tests.log" +PERF_ANALYZER_UNIT_TESTS=../clients/perf_analyzer_unit_tests -backend: "python" -max_batch_size: 1 +RET=0 -input [ - { - name: "IN" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] +rm -f $TEST_LOG -output [ - { - name: "OUT" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] +set +e +$PERF_ANALYZER_UNIT_TESTS >> $TEST_LOG 2>&1 +if [ $? -ne 0 ]; then + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi +set -e -instance_group [ - { - count: 1 - kind: KIND_CPU - } -] - -model_transaction_policy { - decoupled: true -} +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** Test Passed\n***" +else + cat $TEST_LOG + echo -e "\n***\n*** Test FAILED\n***" +fi +exit $RET diff --git a/qa/L0_perf_deeprecommender/run_test.sh b/qa/L0_perf_deeprecommender/run_test.sh index f973d7fea4..434803e7a9 100755 --- a/qa/L0_perf_deeprecommender/run_test.sh +++ b/qa/L0_perf_deeprecommender/run_test.sh @@ -29,9 +29,7 @@ STATIC_BATCH_SIZES=${STATIC_BATCH_SIZES:=1} DYNAMIC_BATCH_SIZES=${DYNAMIC_BATCH_SIZES:=1} INSTANCE_COUNTS=${INSTANCE_COUNTS:=1} -pip3 install perf_analyzer - -PERF_CLIENT=perf_analyzer +PERF_CLIENT=../clients/perf_client REPORTER=../common/reporter.py SERVER=/opt/tritonserver/bin/tritonserver diff --git a/qa/L0_perf_deeprecommender/test.sh b/qa/L0_perf_deeprecommender/test.sh index bfc950aa2b..dc61d56e98 100755 --- a/qa/L0_perf_deeprecommender/test.sh +++ b/qa/L0_perf_deeprecommender/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -88,9 +88,24 @@ if [ $? -ne 0 ]; then fi rm tensorrt_models/deeprecommender_plan/model.onnx +OPTIMIZED_MODEL_NAMES="deeprecommender_graphdef_trt" + +# Create optimized models (TF-TRT and ONNX-TRT) +rm -fr optimized_model_store && mkdir optimized_model_store +for MODEL_NAME in $OPTIMIZED_MODEL_NAMES; do + BASE_MODEL=$(echo ${MODEL_NAME} | cut -d '_' -f 1,2) + cp -r $REPODIR/perf_model_store/${BASE_MODEL} optimized_model_store/${MODEL_NAME} + CONFIG_PATH="optimized_model_store/${MODEL_NAME}/config.pbtxt" + sed -i "s/^name: \"${BASE_MODEL}\"/name: \"${MODEL_NAME}\"/" ${CONFIG_PATH} + echo "optimization { execution_accelerators {" >> ${CONFIG_PATH} + echo "gpu_execution_accelerator : [ {" >> ${CONFIG_PATH} + echo "name : \"tensorrt\" " >> ${CONFIG_PATH} + echo "} ]" >> ${CONFIG_PATH} + echo "}}" >> ${CONFIG_PATH} +done # Tests with each model -for FRAMEWORK in plan onnx libtorch; do +for FRAMEWORK in graphdef plan graphdef_trt onnx libtorch; do MODEL_NAME=${MODEL}_${FRAMEWORK} if [ "$FRAMEWORK" == "plan" ]; then REPO=`pwd`/tensorrt_models @@ -158,7 +173,7 @@ fi rm tensorrt_models/deeprecommender_plan/model.onnx # Tests with each model -for FRAMEWORK in plan onnx libtorch; do +for FRAMEWORK in graphdef plan graphdef_trt onnx libtorch; do MODEL_NAME=${MODEL}_${FRAMEWORK} if [ "$FRAMEWORK" == "plan" ]; then REPO=`pwd`/tensorrt_models diff --git a/qa/L0_perf_kaldi/test.sh b/qa/L0_perf_kaldi/test.sh index 79f5edf7e3..31d4c99ee6 100755 --- a/qa/L0_perf_kaldi/test.sh +++ b/qa/L0_perf_kaldi/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,7 +48,6 @@ pip3 install --upgrade wheel setuptools grpcio-tools # Build client library and kaldi perf client (cd triton-inference-server/build && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX:PATH=/workspace/install && \ make -j16 trtis-clients) diff --git a/qa/L0_perf_nomodel/run_test.sh b/qa/L0_perf_nomodel/run_test.sh index 7a89d5e011..fcfdf16e0e 100755 --- a/qa/L0_perf_nomodel/run_test.sh +++ b/qa/L0_perf_nomodel/run_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -49,20 +49,15 @@ ARCH=${ARCH:="x86_64"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends MODEL_REPO="${PWD}/models" -PERF_CLIENT=perf_analyzer +PERF_CLIENT=../clients/perf_client SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR}" source ../common/util.sh -pip3 install perf_analyzer # DATADIR is already set in environment variable for aarch64 if [ "$ARCH" != "aarch64" ]; then DATADIR="/data/inferenceserver/${REPO_VERSION}" fi -if [ "$SHARED_MEMORY" != "none" ]; then - SERVER_ARGS="${SERVER_ARGS} --allow-client-shm=true" -fi - # Select the single GPU that will be available to the inference server export CUDA_VISIBLE_DEVICES=0 diff --git a/qa/L0_perf_resnet/run_test.sh b/qa/L0_perf_resnet/run_test.sh index 6823525d82..3e7b048c40 100755 --- a/qa/L0_perf_resnet/run_test.sh +++ b/qa/L0_perf_resnet/run_test.sh @@ -53,10 +53,8 @@ rm -fr models && mkdir -p models && \ sed -i "s/^max_batch_size:.*/max_batch_size: ${MAX_BATCH}/" config.pbtxt && \ echo "instance_group [ { count: ${INSTANCE_CNT} }]") -pip3 install perf_analyzer - MEASUREMENT_WINDOW=5000 -PERF_CLIENT=perf_analyzer +PERF_CLIENT=../clients/perf_client # Onnx and onnx-trt models are very slow on Jetson. if [ "$ARCH" == "aarch64" ]; then if [ "$MODEL_FRAMEWORK" == "onnx" ] || [ "$MODEL_FRAMEWORK" == "onnx_trt" ]; then diff --git a/qa/L0_perf_resnet/test.sh b/qa/L0_perf_resnet/test.sh index 8a559ed053..11538db4ce 100755 --- a/qa/L0_perf_resnet/test.sh +++ b/qa/L0_perf_resnet/test.sh @@ -43,12 +43,15 @@ rm -f *.log *.csv *.tjson *.json PROTOCOLS="grpc http triton_c_api" TRT_MODEL_NAME="resnet50_fp32_plan" +TF_MODEL_NAME="resnet50v1.5_fp16_savedmodel" PYT_MODEL_NAME="resnet50_fp32_libtorch" ONNX_MODEL_NAME="resnet50_fp32_onnx" # The base model name should be the prefix to the # respective optimized model name. +TFTRT_MODEL_NAME="resnet50v1.5_fp16_savedmodel_trt" ONNXTRT_MODEL_NAME="resnet50_fp32_onnx_trt" +TFAMP_MODEL_NAME="resnet50v1.5_fp16_savedmodel_amp" ARCH=${ARCH:="x86_64"} REPODIR=${REPODIR:="/data/inferenceserver/${REPO_VERSION}"} @@ -64,10 +67,15 @@ STATIC_BATCH=1 INSTANCE_CNT=1 CONCURRENCY=1 -MODEL_NAMES="${TRT_MODEL_NAME} ${ONNX_MODEL_NAME} ${PYT_MODEL_NAME}" - -OPTIMIZED_MODEL_NAMES="${ONNXTRT_MODEL_NAME}" +MODEL_NAMES="${TRT_MODEL_NAME} ${TF_MODEL_NAME} ${ONNX_MODEL_NAME} ${PYT_MODEL_NAME}" +# Disable TF-TRT test on Jetson due to Segfault +# Disable ORT-TRT test on Jetson due to support being disabled +if [ "$ARCH" == "aarch64" ]; then + OPTIMIZED_MODEL_NAMES="${TFAMP_MODEL_NAME}" +else + OPTIMIZED_MODEL_NAMES="${TFTRT_MODEL_NAME} ${TFAMP_MODEL_NAME} ${ONNXTRT_MODEL_NAME}" +fi # Create optimized models rm -fr optimized_model_store && mkdir optimized_model_store @@ -78,15 +86,21 @@ for MODEL_NAME in $OPTIMIZED_MODEL_NAMES; do sed -i "s/^name: \"${BASE_MODEL}\"/name: \"${MODEL_NAME}\"/" ${CONFIG_PATH} echo "optimization { execution_accelerators {" >> ${CONFIG_PATH} echo "gpu_execution_accelerator : [ {" >> ${CONFIG_PATH} - echo "name : \"tensorrt\" " >> ${CONFIG_PATH} - - if [ "${MODEL_NAME}" = "${ONNXTRT_MODEL_NAME}" ] ; then - echo "parameters { key: \"precision_mode\" value: \"FP16\" }" >> ${CONFIG_PATH} - echo "parameters { key: \"max_workspace_size_bytes\" value: \"1073741824\" }" >> ${CONFIG_PATH} - echo "parameters { key: \"trt_engine_cache_enable\" value: \"1\" }" >> ${CONFIG_PATH} - echo "parameters { key: \"trt_engine_cache_path\" value: \"${CACHE_PATH}\" } " >> ${CONFIG_PATH} + if [ "${MODEL_NAME}" = "${TFAMP_MODEL_NAME}" ] ; then + echo "name : \"auto_mixed_precision\" " >> ${CONFIG_PATH} + else + echo "name : \"tensorrt\" " >> ${CONFIG_PATH} + if [ "${MODEL_NAME}" = "${TFTRT_MODEL_NAME}" ] ; then + echo "parameters { key: \"precision_mode\" value: \"FP16\" }" >> ${CONFIG_PATH} + fi + + if [ "${MODEL_NAME}" = "${ONNXTRT_MODEL_NAME}" ] ; then + echo "parameters { key: \"precision_mode\" value: \"FP16\" }" >> ${CONFIG_PATH} + echo "parameters { key: \"max_workspace_size_bytes\" value: \"1073741824\" }" >> ${CONFIG_PATH} + echo "parameters { key: \"trt_engine_cache_enable\" value: \"1\" }" >> ${CONFIG_PATH} + echo "parameters { key: \"trt_engine_cache_path\" value: \"${CACHE_PATH}\" } " >> ${CONFIG_PATH} + fi fi - echo "} ]" >> ${CONFIG_PATH} echo "}}" >> ${CONFIG_PATH} done @@ -199,4 +213,26 @@ for MODEL_NAME in $OPTIMIZED_MODEL_NAMES; do ARCH=${ARCH} \ bash -x run_test.sh done -done \ No newline at end of file +done + +# FIXME Disable the following due to +# https://jirasw.nvidia.com/browse/DLIS-2933. +# +# Needs this additional test configuration for comparing against TFS. +if [ "$ARCH" == "x86_64" ]; then + MODEL_NAME=${TF_MODEL_NAME} + REPO=$REPODIR/perf_model_store + STATIC_BATCH=128 + INSTANCE_CNT=1 + CONCURRENCY=1 + FRAMEWORK=$(echo ${MODEL_NAME} | cut -d '_' -f 3) + MODEL_NAME=${MODEL_NAME} \ + MODEL_FRAMEWORK=${FRAMEWORK} \ + MODEL_PATH="$REPO/${MODEL_NAME}" \ + STATIC_BATCH=${STATIC_BATCH} \ + PERF_CLIENT_PROTOCOL="grpc" \ + INSTANCE_CNT=${INSTANCE_CNT} \ + CONCURRENCY=${CONCURRENCY} \ + ARCH=${ARCH} \ + bash -x run_test.sh +fi diff --git a/qa/L0_perf_tensorrt_llm/test.sh b/qa/L0_perf_tensorrt_llm/test.sh index 5bf418c5c5..e74b01e568 100755 --- a/qa/L0_perf_tensorrt_llm/test.sh +++ b/qa/L0_perf_tensorrt_llm/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -35,7 +35,7 @@ MODEL_NAME="gpt2_tensorrt_llm" NAME="tensorrt_llm_benchmarking_test" MODEL_REPOSITORY="$(pwd)/triton_model_repo" TENSORRTLLM_BACKEND_DIR="/workspace/tensorrtllm_backend" -GPT_DIR="$TENSORRTLLM_BACKEND_DIR/tensorrt_llm/examples/models/core/gpt" +GPT_DIR="$TENSORRTLLM_BACKEND_DIR/tensorrt_llm/examples/gpt" TOKENIZER_DIR="$GPT_DIR/gpt2" ENGINES_DIR="${BASE_DIR}/engines/inflight_batcher_llm/${NUM_GPUS}-gpu" TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} @@ -43,7 +43,13 @@ SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends SERVER_LOG="${NAME}_server.log" SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} -source ../common/trtllm_util.sh + +function clone_tensorrt_llm_backend_repo { + rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR + apt-get update && apt-get install git-lfs -y --no-install-recommends + git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR + cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive +} # Update Open MPI to a version compatible with SLURM. function upgrade_openmpi { @@ -89,6 +95,125 @@ function upgrade_openmpi { mpirun --version } +function build_gpt2_base_model { + # Download weights from HuggingFace Transformers + cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2 + rm pytorch_model.bin model.safetensors + if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then + echo "Downloading pytorch_model.bin failed." + exit 1 + fi + cd ${GPT_DIR} + + # Convert weights from HF Tranformers to FT format + python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" + cd ${BASE_DIR} +} + +function build_gpt2_tensorrt_engine { + # Build TensorRT engines + cd ${GPT_DIR} + trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ + --gpt_attention_plugin float16 \ + --remove_input_padding enable \ + --paged_kv_cache enable \ + --gemm_plugin float16 \ + --workers "${NUM_GPUS}" \ + --output_dir "${ENGINES_DIR}" + + cd ${BASE_DIR} +} + +function replace_config_tags { + tag_to_replace="${1}" + new_value="${2}" + config_file_path="${3}" + sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path} +} + +function prepare_model_repository { + rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} + cp -r ${TENSORRTLLM_BACKEND_DIR}/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} + rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls + mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" + + replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" +} + +# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on +# success, 1 on failure +function wait_for_server_ready() { + local wait_time_secs="${1:-30}" + shift + local spids=("$@") + + WAIT_RET=0 + + for _ in $(seq "$wait_time_secs"); do + for pid in "${spids[@]}"; do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "=== Server not running." + WAIT_RET=1 + return + fi + done + + sleep 1 + + if curl -s --fail localhost:8000/v2/health/ready && + curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then + return + fi + done + + echo "=== Timeout $wait_time_secs secs. Server not ready." + WAIT_RET=1 +} + +function run_server { + python3 ${TENSORRTLLM_BACKEND_DIR}/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & + sleep 2 # allow time to obtain the pid(s) + # Read PIDs into an array, trimming whitespaces + readarray -t SERVER_PID < <(pgrep "tritonserver") + + wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" + if [ "$WAIT_RET" != "0" ]; then + # Cleanup + kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi +} + +function kill_server { + pgrep tritonserver | xargs kill -SIGINT + for pid in "${SERVER_PID[@]}"; do + echo "Waiting for proc ${pid} to terminate..." + while kill -0 $pid >/dev/null 2>&1; do + sleep 1 + done + done +} + upgrade_openmpi clone_tensorrt_llm_backend_repo build_gpt2_base_model diff --git a/qa/L0_perf_vllm/test.sh b/qa/L0_perf_vllm/test.sh index 263a4fc3db..e1ce8cf2ed 100755 --- a/qa/L0_perf_vllm/test.sh +++ b/qa/L0_perf_vllm/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,13 +41,13 @@ SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR} export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:=0} EXPORT_FILE=profile-export-vllm-model.json -pip3 install tritonclient[perf_analyzer] - +pip3 install tritonclient rm -rf $MODEL_REPO $EXPORT_FILE *.tjson *.json *.csv mkdir -p $MODEL_REPO/$MODEL_NAME/1 echo '{ "model":"gpt2", + "disable_log_requests": "true", "gpu_memory_utilization": 0.5 }' >$MODEL_REPO/$MODEL_NAME/1/model.json diff --git a/qa/L0_pinned_memory/test.sh b/qa/L0_pinned_memory/test.sh index beb16cd132..48ec05b84c 100755 --- a/qa/L0_pinned_memory/test.sh +++ b/qa/L0_pinned_memory/test.sh @@ -38,12 +38,10 @@ if [ ! -z "$TEST_REPO_ARCH" ]; then REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} fi -pip3 install perf_analyzer - # Use "--request-count" throughout the test to PA stability criteria and # reduce flaky failures from PA unstable measurements. REQUEST_COUNT=10 -CLIENT=perf_analyzer +CLIENT=../clients/perf_client # Only use libtorch as it accepts GPU I/O and it can handle variable shape BACKENDS=${BACKENDS:="libtorch"} diff --git a/qa/L0_query/test.sh b/qa/L0_query/test.sh index ac9e3a4d3d..153cd69381 100755 --- a/qa/L0_query/test.sh +++ b/qa/L0_query/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -69,7 +69,7 @@ export TEST_FAIL_WITH_QUERY_RESULT=1 export TEST_BYTE_SIZE=4 SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models --allow-client-shm=true" +SERVER_ARGS="--model-repository=`pwd`/models" SERVER_LOG="./inference_server.log" run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/L0_request_cancellation/grpc_cancellation_test.py b/qa/L0_request_cancellation/grpc_cancellation_test.py index 253c23ab02..7d9331d5f5 100755 --- a/qa/L0_request_cancellation/grpc_cancellation_test.py +++ b/qa/L0_request_cancellation/grpc_cancellation_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -202,34 +202,12 @@ def test_grpc_async_infer_response_complete_during_cancellation(self): ) # ensure the cancellation is processed self._assert_callback_cancelled() - def test_grpc_async_infer_cancellation_before_finish_0(self): - # First version of test_grpc_async_infer_cancellation_before_finish - # Cancellation notification is processed before the final response state. + def test_grpc_async_infer_cancellation_during_response_complete(self): # long test - self.test_duration_delta = 2 + self.test_duration_delta = 2.5 delay_notification_sec = ( int(os.getenv("TRITONSERVER_DELAY_GRPC_NOTIFICATION")) / 1000 ) - future = self._client.async_infer( - model_name=self._model_name, - inputs=self._inputs, - callback=self._callback, - outputs=self._outputs, - ) - # ensure the cancellation is received between InferResponseComplete checking cancellation and Finish - time.sleep(self._model_delay + 2) - future.cancel() - time.sleep(delay_notification_sec + 1) # ensure the cancellation is processed - self._assert_callback_cancelled() - - def test_grpc_async_infer_cancellation_before_finish_1(self): - # Second version of test_grpc_async_infer_cancellation_before_finish - # Cancellation notification is processed after the final response state. - # long test - self.test_duration_delta = 2 - delay_process_entry_sec = ( - int(os.getenv("TRITONSERVER_DELAY_GRPC_PROCESS_ENTRY")) / 1000 - ) delay_response_completion_sec = ( int(os.getenv("TRITONSERVER_DELAY_RESPONSE_COMPLETION")) / 1000 ) @@ -240,38 +218,13 @@ def test_grpc_async_infer_cancellation_before_finish_1(self): outputs=self._outputs, ) # ensure the cancellation is received between InferResponseComplete checking cancellation and Finish - time.sleep(self._model_delay + delay_process_entry_sec + 2) + time.sleep(self._model_delay + 2) future.cancel() time.sleep( - delay_response_completion_sec + delay_notification_sec + delay_response_completion_sec ) # ensure the cancellation is processed self._assert_callback_cancelled() - def test_grpc_async_infer_cancellation_before_response_complete_and_process_after_final_response( - self, - ): - # Received cancellation before InferResponseComplete and the notification - # state is processed after processing final response state. - # long test - self.test_duration_delta = 2 - delay_notification_sec = ( - int(os.getenv("TRITONSERVER_DELAY_GRPC_NOTIFICATION")) / 1000 - ) - delay_response_complete_exec_sec = ( - int(os.getenv("TRITONSERVER_DELAY_RESPONSE_COMPLETE_EXEC")) / 1000 - ) - future = self._client.async_infer( - model_name=self._model_name, - inputs=self._inputs, - callback=self._callback, - outputs=self._outputs, - ) - # ensure the cancellation is received before InferResponseComplete checking cancellation - time.sleep(self._model_delay + 2) - future.cancel() - time.sleep(delay_notification_sec + 1) # ensure the cancellation is processed - self._assert_callback_cancelled() - if __name__ == "__main__": unittest.main() diff --git a/qa/L0_request_cancellation/test.sh b/qa/L0_request_cancellation/test.sh index 2b92c12027..7789f2006a 100755 --- a/qa/L0_request_cancellation/test.sh +++ b/qa/L0_request_cancellation/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,10 +42,8 @@ export CUDA_VISIBLE_DEVICES=0 SERVER=/opt/tritonserver/bin/tritonserver source ../common/util.sh -CANCEL_LOG_LINE="Cancellation notification received for " RET=0 -rm -f *.log # # Unit tests @@ -68,7 +66,7 @@ if [ $? -ne 0 ]; then fi # -# Python gRPC cancellation tests +# gRPC cancellation tests # rm -rf models && mkdir models mkdir -p models/custom_identity_int32/1 && (cd models/custom_identity_int32 && \ @@ -86,9 +84,7 @@ for TEST_CASE in "test_grpc_async_infer" \ "test_aio_grpc_stream_infer" \ "test_grpc_async_infer_cancellation_at_step_start" \ "test_grpc_async_infer_response_complete_during_cancellation" \ - "test_grpc_async_infer_cancellation_before_finish_0" \ - "test_grpc_async_infer_cancellation_before_finish_1" \ - "test_grpc_async_infer_cancellation_before_response_complete_and_process_after_final_response"; do + "test_grpc_async_infer_cancellation_during_response_complete"; do TEST_LOG="./grpc_cancellation_test.$TEST_CASE.log" SERVER_LOG="grpc_cancellation_test.$TEST_CASE.server.log" if [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_at_step_start" ]; then @@ -96,18 +92,12 @@ for TEST_CASE in "test_grpc_async_infer" \ elif [ "$TEST_CASE" == "test_grpc_async_infer_response_complete_during_cancellation" ]; then export TRITONSERVER_DELAY_GRPC_NOTIFICATION=5000 export TRITONSERVER_DELAY_GRPC_ENQUEUE=5000 - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_finish_0" ]; then + elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_during_response_complete" ]; then export TRITONSERVER_DELAY_GRPC_NOTIFICATION=5000 export TRITONSERVER_DELAY_RESPONSE_COMPLETION=5000 - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_finish_1" ]; then - export TRITONSERVER_DELAY_GRPC_PROCESS_ENTRY=1000 - export TRITONSERVER_DELAY_RESPONSE_COMPLETION=5000 - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_response_complete_and_process_after_final_response" ]; then - export TRITONSERVER_DELAY_GRPC_NOTIFICATION=5000 - export TRITONSERVER_DELAY_RESPONSE_COMPLETE_EXEC=5000 fi - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=2" + SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" run_server if [ "$SERVER_PID" == "0" ]; then echo -e "\n***\n*** Failed to start $SERVER\n***" @@ -123,7 +113,7 @@ for TEST_CASE in "test_grpc_async_infer" \ RET=1 fi - count=$(grep -o "$CANCEL_LOG_LINE" $SERVER_LOG | wc -l) + count=$(grep -o "Cancellation notification received for" $SERVER_LOG | wc -l) if [ $count == 0 ]; then echo -e "\n***\n*** Cancellation not received by server on $TEST_CASE\n***" cat $SERVER_LOG @@ -133,23 +123,6 @@ for TEST_CASE in "test_grpc_async_infer" \ cat $SERVER_LOG RET=1 fi - - # Tests "test_grpc_async_infer" and "test_aio_grpc_async_infer" ends - # prematurely before state is released. - if [[ "$TEST_CASE" != "test_grpc_async_infer" && "$TEST_CASE" != "test_aio_grpc_async_infer" ]]; then - count=$(grep -o "StateRelease" $SERVER_LOG | wc -l) - state_released=${state_released:=1} - if [ $count == 0 ]; then - echo -e "\n***\n*** State not released by server on $TEST_CASE\n***" - cat $SERVER_LOG - RET=1 - elif [ $count -ne $state_released ]; then - echo -e "\n***\n*** Unexpected states released by server on $TEST_CASE. Expected $state_released but released $count.\n***" - cat $SERVER_LOG - RET=1 - fi - unset state_released - fi set -e kill $SERVER_PID @@ -160,89 +133,9 @@ for TEST_CASE in "test_grpc_async_infer" \ elif [ "$TEST_CASE" == "test_grpc_async_infer_response_complete_during_cancellation" ]; then unset TRITONSERVER_DELAY_GRPC_NOTIFICATION unset TRITONSERVER_DELAY_GRPC_ENQUEUE - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_finish_0" ]; then + elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_during_response_complete" ]; then unset TRITONSERVER_DELAY_GRPC_NOTIFICATION unset TRITONSERVER_DELAY_RESPONSE_COMPLETION - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_finish_1" ]; then - unset TRITONSERVER_DELAY_GRPC_PROCESS_ENTRY - unset TRITONSERVER_DELAY_RESPONSE_COMPLETION - elif [ "$TEST_CASE" == "test_grpc_async_infer_cancellation_before_response_complete_and_process_after_final_response" ]; then - unset TRITONSERVER_DELAY_GRPC_NOTIFICATION - unset TRITONSERVER_DELAY_RESPONSE_COMPLETE_EXEC - fi -done - -# -# C++ gRPC cancellation tests -# -# allow_timeout_override disables queue prefetching, keeping requests queued -# long enough for the "Queued" cancellation tests to cancel them before -# forwarding to the rate limiter. This saves overall test time. -cat >> models/custom_identity_int32/config.pbtxt <<'EOF' -dynamic_batching { - default_queue_policy { - allow_timeout_override: true - } -} -EOF - -GRPC_CANCELLATION_TEST_CPP=../clients/grpc_cancellation_test - -for ENTRY in "TestGrpcAsyncInferCancelExecutingRequest 1" \ - "TestGrpcAsyncInferCancelQueuedRequest 2" \ - "TestGrpcAsyncInferCancelAfterCompletionIsNoOp 0" \ - "TestGrpcAsyncInferWithoutContextStillCompletes 0" \ - "TestGrpcAsyncInferMultiCancelExecutingRequests 2" \ - "TestGrpcAsyncInferMultiCancelQueuedRequest 2" \ - "TestGrpcStreamInferCancelExecutingRequest 1" \ - "TestGrpcStreamInferCancelQueuedRequest 1" \ - "TestGrpcStreamCancelWithoutInfer 1" \ - "TestGrpcStreamCancelThenRestart 1"; do - read -r TEST_CASE EXPECTED_CANCEL_COUNT <<< "$ENTRY" - - TEST_LOG="./grpc_cancellation_test_cpp.$TEST_CASE.log" - SERVER_LOG="./grpc_cancellation_test_cpp.$TEST_CASE.server.log" - - # AsyncInferMulti fans out N concurrent requests; bump to 3 CPU instances - # so each can execute in parallel. Every other test uses the default - # single-instance config. - if [ "$TEST_CASE" == "TestGrpcAsyncInferMultiCancelExecutingRequests" ]; then - sed -i 's|instance_group .*|instance_group [{ count: 3, kind: KIND_CPU }]|' \ - models/custom_identity_int32/config.pbtxt - fi - - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=2" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH \ - $GRPC_CANCELLATION_TEST_CPP \ - --gtest_filter="GrpcCancellationTest.$TEST_CASE" > $TEST_LOG 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** C++ gRPC Cancellation Tests Failed on $TEST_CASE\n***" - cat $TEST_LOG - RET=1 - fi - - cancel_count=$(grep -c "$CANCEL_LOG_LINE" $SERVER_LOG || true) - if [ $cancel_count -ne $EXPECTED_CANCEL_COUNT ]; then - echo -e "\n***\n*** Unexpected cancellation count on $TEST_CASE. Expected $EXPECTED_CANCEL_COUNT but received $cancel_count.\n***" - cat $SERVER_LOG - RET=1 - fi - set -e - - kill $SERVER_PID - wait $SERVER_PID - - if [ "$TEST_CASE" == "TestGrpcAsyncInferMultiCancelExecutingRequests" ]; then - sed -i 's|instance_group .*|instance_group [{ kind: KIND_CPU }]|' \ - models/custom_identity_int32/config.pbtxt fi done diff --git a/qa/L0_response_cache/generate_random_data.py b/qa/L0_response_cache/generate_random_data.py deleted file mode 100755 index 27e8c76c39..0000000000 --- a/qa/L0_response_cache/generate_random_data.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse -import json - -import numpy as np - - -def generate_input_data(num_inputs, batch_size, output_file): - data = {"data": []} - for _ in range(num_inputs): - input_data = np.random.rand(batch_size, 1024).astype(np.float32) - entry = {"INPUT0": input_data.flatten().tolist()} - data["data"].append(entry) - - with open(output_file, "w") as f: - json.dump(data, f) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Generate random input data for perf_analyzer." - ) - parser.add_argument( - "--num-inputs", type=int, help="Number of unique random inputs to generate." - ) - parser.add_argument("--batch-size", type=int, help="The batch size for each input.") - parser.add_argument( - "--output-file", type=str, help="The name of the output JSON file." - ) - args = parser.parse_args() - - generate_input_data(args.num_inputs, args.batch_size, args.output_file) - print(f"Successfully generated {args.num_inputs} inputs in '{args.output_file}'.") diff --git a/qa/L0_response_cache/response_cache_test b/qa/L0_response_cache/response_cache_test new file mode 100755 index 0000000000..d3807fac01 Binary files /dev/null and b/qa/L0_response_cache/response_cache_test differ diff --git a/qa/L0_response_cache/test.sh b/qa/L0_response_cache/test.sh index bb372dab9f..c99aac89d1 100755 --- a/qa/L0_response_cache/test.sh +++ b/qa/L0_response_cache/test.sh @@ -381,107 +381,6 @@ ERROR_MESSAGE="\n***\n*** Failed: Request added to cache successfully when it wa CACHE_SIZE=200 test_response_cache_ensemble_model "${TEST_NAME}" "${ERROR_MESSAGE}" - -############### Response Cache Memory Growth Test ############### - -# Set server, client and valgrind arguments -LEAKCHECK=/usr/bin/valgrind -MASSIF_TEST=../common/check_massif_log.py -MODEL="identity_cache" -LEAKCHECK_LOG="${MODEL}.valgrind.log" -MASSIF_LOG="${MODEL}.valgrind.massif" -GRAPH_LOG="memory_growth_${MODEL}.log" -SERVER_LOG="${MODEL}.server.log" -CLIENT_LOG="${MODEL}_PA.client.log" -RANDOM_DATA_CLIENT_LOG="${MODEL}_random_data_script.log" -RANDOM_DATA_JSON="`pwd`/random_inputs.json" -RANDOM_DATA_GENERATOR="generate_random_data.py" - -LEAKCHECK_ARGS="--tool=massif --time-unit=B --massif-out-file=$MASSIF_LOG --max-threads=3000 --log-file=$LEAKCHECK_LOG" -SERVER_ARGS="--model-repository=`pwd`/models --model-control-mode=explicit --load-model=${MODEL} --cache-config=local,size=10485760" # 10MB cache - -set +e -# Generate random data for perf_analyzer requests to fill the cache and maximize cache misses -python "$RANDOM_DATA_GENERATOR" --num-inputs=10000 --batch-size=1 --output-file="${RANDOM_DATA_JSON}" >> "$RANDOM_DATA_CLIENT_LOG" 2>&1 -if [ $? -ne 0 ]; then - cat "$RANDOM_DATA_CLIENT_LOG" - echo -e "\n***\n*** Failed to run ${RANDOM_DATA_GENERATOR}.\n***" - RET=1 - exit 1 -else - # Check if the JSON data file was generated - if [ ! -f "${RANDOM_DATA_JSON}" ]; then - echo -e "\n***\n*** FAILED - JSON data file was not found at the expected path: ${RANDOM_DATA_JSON}\n***" - RET=1 - exit 1 - fi -fi -set -e - -# Run the server -run_server_leakcheck -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -pip3 install perf_analyzer - -TEMP_RET=0 -REPETITION=10 -CONCURRENCY=20 -CLIENT_BS=1 -PERF_ANALYZER=perf_analyzer -TEMP_CLIENT_LOG=temp_client.log - -set +e -SECONDS=0 -# Run the perf analyzer 'REPETITION' times -for ((i=1; i<=$REPETITION; i++)); do - # Use random data to ensure cache misses - $PERF_ANALYZER -v -m $MODEL --shape=INPUT0:1024 -i grpc --concurrency-range $CONCURRENCY -b $CLIENT_BS -p 20000 --input-data="${RANDOM_DATA_JSON}" > $TEMP_CLIENT_LOG 2>&1 - PA_RET=$? - cat $TEMP_CLIENT_LOG >> $CLIENT_LOG - # Success - if [ ${PA_RET} -eq 0 ]; then - continue - # Unstable measurement: OK for this test - elif [ ${PA_RET} -eq 2 ]; then - continue - # Other failures unexpected, report error - else - echo -e "\n***\n*** perf_analyzer for $MODEL failed on iteration $i\n***" >> $CLIENT_LOG - RET=1 - fi -done -TEST_DURATION=$SECONDS -set -e - -# Stop Server -kill $SERVER_PID -wait $SERVER_PID - -set +e - -# Log test duration and the graph for memory growth -MAX_ALLOWED_ALLOC=2 # MB -hrs=$(printf "%02d" $((TEST_DURATION / 3600))) -mins=$(printf "%02d" $(((TEST_DURATION / 60) % 60))) -secs=$(printf "%02d" $((TEST_DURATION % 60))) -echo -e "Test Duration: $hrs:$mins:$secs (HH:MM:SS)" >> ${GRAPH_LOG} -ms_print ${MASSIF_LOG} | head -n35 >> ${GRAPH_LOG} -cat ${GRAPH_LOG} -# Check the massif output -python $MASSIF_TEST $MASSIF_LOG $MAX_ALLOWED_ALLOC --start-from-middle >> $GRAPH_LOG 2>&1 -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Memory growth test for $MODEL Failed.\n***" - RET=1 -fi -# Always output memory usage for easier triage of MAX_ALLOWED_ALLOC settings in the future -grep -i "Change in memory allocation" "${GRAPH_LOG}" || true -set -e - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else diff --git a/qa/L0_sagemaker/sagemaker_request_many_chunks.py b/qa/L0_sagemaker/sagemaker_request_many_chunks.py deleted file mode 100755 index e7ef8dd686..0000000000 --- a/qa/L0_sagemaker/sagemaker_request_many_chunks.py +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/python -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import socket -import sys -import unittest - -sys.path.append("../common") -from test_util import MIB, get_server_process_from_env, wait_for_stable_rss - - -class SagemakerRequestManyChunksTest(unittest.TestCase): - def setUp(self): - self._local_host = "localhost" - self._sagemaker_port = 8080 - # Must match server kMaxChunkedChunks (http_server.cc). - self._k_max_chunked_chunks = 65536 - self._over_max_chunks_error = f"Chunked request body exceeds maximum of {self._k_max_chunked_chunks} non-empty chunks. Send fewer or larger HTTP chunks." - - def _sagemaker_chunked_header(self): - return ( - f"POST /models HTTP/1.1\r\n" f"X-Amzn-SageMaker-Target-Model: ZZZZZZZ\r\n" - ) - - def send_chunked_request( - self, - header: str, - chunk_count: int, - expected_response: str, - expected_http_status=400, - ): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - header = ( - f"{header}" - f"Host: {self._local_host}:{self._sagemaker_port}\r\n" - f"Content-Type: application/octet-stream\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ) - try: - s.connect((self._local_host, self._sagemaker_port)) - # HTTP request with chunked encoding - s.sendall((header.encode())) - - # Send chunked payload (server may close early when over chunk limit) - for _ in range(chunk_count): - try: - s.send(b"1\r\nA\r\n") - except (BrokenPipeError, ConnectionResetError): - break - try: - s.sendall(b"0\r\n\r\n") - except (BrokenPipeError, ConnectionResetError): - # Server may close/reset early after detecting chunk-limit violation. - # In that case, failing to send the terminating chunk is expected. - pass - - # Receive response - response = b"" - while True: - try: - chunk = s.recv(4096) - if not chunk: - break - response += chunk - except ConnectionResetError: - break - except socket.timeout: - break - self.assertTrue( - response, - "expected error response body, but socket closed/reset before any bytes", - ) - status_line = response.split(b"\r\n", 1)[0].decode(errors="replace") - self.assertTrue( - status_line.startswith(f"HTTP/1.1 {expected_http_status} "), - f"expected HTTP status {expected_http_status}, got {status_line!r}", - ) - self.assertIn(expected_response, response.decode()) - except Exception as e: - raise (e) - finally: - s.close() - - def test_chunked_at_max_chunks(self): - self.send_chunked_request( - self._sagemaker_chunked_header(), - self._k_max_chunked_chunks, - "failed to parse the request JSON buffer: Invalid value. at 0", - ) - - def test_chunked_rejected_over_max_chunks(self): - self.send_chunked_request( - self._sagemaker_chunked_header(), - self._k_max_chunked_chunks + 1, - self._over_max_chunks_error, - ) - - def test_chunked_over_max_chunks_reject_with_bounded_rss_growth(self): - many_chunks = 1000000 - - # verify server is running - server = get_server_process_from_env("SERVER_PID") - self.assertTrue(server.is_running()) - - # warm up and wait until RSS is stable. - self.send_chunked_request( - self._sagemaker_chunked_header(), - 1000000, # way over max chunks - self._over_max_chunks_error, - ) - # Wait until RSS is stable across several measurements before continuing. - server = get_server_process_from_env("SERVER_PID") - wait_for_stable_rss(server) - - # Monitor RSS growth over 100 requests. - repeat_request_count = 100 - rss_before = server.memory_info().rss - # TODO: Why sagemaker server occasionally grows >10 MiB but http server always smaller than 1 MiB? - max_rss_growth_bytes = 20 * MIB - - for _ in range(repeat_request_count): - self.send_chunked_request( - self._sagemaker_chunked_header(), - many_chunks, # way over max chunks - self._over_max_chunks_error, - ) - - rss_after = server.memory_info().rss - growth = rss_after - rss_before - print( - f"RSS: before={rss_before / MIB:.1f} MiB, " - f"after={rss_after / MIB:.1f} MiB, " - f"growth={growth / MIB:.1f} MiB, " - f"limit={max_rss_growth_bytes / MIB:.0f} MiB", - flush=True, - ) - self.assertLess( - growth, - max_rss_growth_bytes, - f"Server RSS grew by {growth / MIB:.1f} MiB after " - f"{repeat_request_count} over-limit chunked infer requests " - f"(limit {max_rss_growth_bytes / MIB:.0f} MiB).", - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_sagemaker/test.sh b/qa/L0_sagemaker/test.sh index 61ed1b3b48..dde4c794d8 100755 --- a/qa/L0_sagemaker/test.sh +++ b/qa/L0_sagemaker/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -327,7 +327,7 @@ fi # Helper library to parse SSE events # https://github.com/mpetazzoni/sseclient -pip install sseclient-py psutil +pip install sseclient-py # Inference with generate_stream inference type set +e @@ -554,14 +554,6 @@ else RET=1 fi fi - -# Verify that loading without X-Amzn-SageMaker-Target-Model header logs the model name. -grep "Loading SageMaker TargetModel: sm_mme_model_1" $SERVER_LOG -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected TargetModel log with model name\n***" - cat $SERVER_LOG - RET=1 -fi set -e unset SAGEMAKER_MULTI_MODEL @@ -573,255 +565,8 @@ kill $SERVER_PID wait $SERVE_PID # MME end -### Test Sagemaker Requests Containing Many Chunks ### -rm -rf models && mkdir models && \ - cp -r $DATADIR/qa_model_repository/onnx_int32_int32_int32 models/sm_model && \ - rm -r models/sm_model/2 && rm -r models/sm_model/3 && \ - sed -i "s/onnx_int32_int32_int32/sm_model/" models/sm_model/config.pbtxt - -export SAGEMAKER_TRITON_DEFAULT_MODEL_NAME=sm_model -REQUEST_MANY_CHUNKS_PY="sagemaker_request_many_chunks.py" -CLIENT_LOG="./client.sagemaker_request_many_chunks.log" -SERVER_LOG="./server.sagemaker_request_many_chunks.log" - -serve > $SERVER_LOG 2>&1 & -SERVE_PID=$! -# Obtain Triton PID in such way as $! will return the script PID -sleep 1 -SERVER_PID=`ps | grep tritonserver | awk '{ printf $1 }'` -sagemaker_wait_for_server_ready $SERVER_PID 10 -if [ "$WAIT_RET" != "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - kill $SERVER_PID || true - cat $SERVER_LOG - exit 1 -fi - -# Ping -set +e -code=`curl -s -w %{http_code} -o ./ping.out localhost:8080/ping` -set -e -if [ "$code" != "200" ]; then - cat ./ping.out - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -set +e -SERVER_PID=$SERVER_PID python $REQUEST_MANY_CHUNKS_PY >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Sagemaker Request Many Chunks Test Failed\n***" - cat $SERVER_LOG - cat $CLIENT_LOG - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVE_PID - -### Restricted API regression for SageMaker endpoint ### -# Verify that --http-restricted-api applies to SageMaker model management -# endpoints (load, unload, list, get) while leaving health and inference -# unrestricted. - -SERVER_LOG="./sagemaker_restricted_api_server.log" -SERVER_ARGS="--allow-sagemaker=true --allow-http=true \ - --allow-grpc=false --allow-metrics=false \ - --model-repository=`pwd`/models \ - --model-control-mode=explicit \ - --load-model=sm_model \ - --http-restricted-api=model-repository:X-SM-Auth=secret" -run_server_nowait -sagemaker_wait_for_server_ready $SERVER_PID 10 -if [ "$WAIT_RET" != "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - kill $SERVER_PID || true - cat $SERVER_LOG - exit 1 -fi - -set +e - -# Health should succeed without restricted header -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out localhost:8080/ping` -if [ "$code" != "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected /ping to succeed without restricted header\n***" - RET=1 -fi - -# Inference should succeed without restricted header -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8080/invocations \ - -H "Content-Type: application/json" \ - -d '{"inputs":[{"name":"INPUT0","datatype":"INT32","shape":[1,16],"data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},{"name":"INPUT1","datatype":"INT32","shape":[1,16],"data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}]}'` -if [ "$code" != "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected /invocations inference to succeed without restricted header\n***" - RET=1 -fi - -# List models without auth header should be blocked -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out localhost:8080/models` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected GET /models to return 403 without restricted header (got $code)\n***" - RET=1 -else - grep "This API is restricted" ./curl.out - if [ $? -ne 0 ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected restriction error message in response body\n***" - RET=1 - fi -fi - -# Get model without auth header should be blocked -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out localhost:8080/models/sm_model` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected GET /models/ to return 403 without restricted header (got $code)\n***" - RET=1 -fi - -# Load model without auth header should be blocked -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8080/models \ - -H "Content-Type: application/json" \ - -d '{"model_name":"test","url":"/opt/ml/models/123/model"}'` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected POST /models (load) to return 403 without restricted header (got $code)\n***" - RET=1 -fi - -# Unload model without auth header should be blocked -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X DELETE localhost:8080/models/sm_model` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected DELETE /models (unload) to return 403 without restricted header (got $code)\n***" - RET=1 -fi - -# List models WITH correct auth header should succeed -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -H "X-SM-Auth: secret" localhost:8080/models` -if [ "$code" == "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected GET /models with auth header to pass restriction check\n***" - RET=1 -fi - -# Get model WITH correct auth header should pass restriction check -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -H "X-SM-Auth: secret" localhost:8080/models/sm_model` -if [ "$code" == "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected GET /models/ with auth header to pass restriction check\n***" - RET=1 -fi - -# Wrong auth header value should be rejected -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -H "X-SM-Auth: wrong" localhost:8080/models` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected wrong auth header value to return 403 (got $code)\n***" - RET=1 -fi - -# Verify core HTTP endpoint is also restricted by the same flag -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/repository/index` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected core HTTP repository index to be restricted (got $code)\n***" - RET=1 -fi - -set -e - -kill $SERVER_PID -wait $SERVER_PID - -### HTTP max input size enforcement on SageMaker endpoint ### -# Verify that --http-max-input-size is enforced on the SageMaker /invocations -# path, not just the core HTTP endpoint. - -rm -rf models_identity -mkdir -p models_identity/sm_identity/1 && \ - cp ../python_models/identity_fp32/model.py models_identity/sm_identity/1/ && \ - cp ../python_models/identity_fp32/config.pbtxt models_identity/sm_identity/ && \ - sed -i "s/identity_fp32/sm_identity/" models_identity/sm_identity/config.pbtxt -mkdir -p /opt/ml -ln -sf `pwd`/models_identity /opt/ml/model - -export SAGEMAKER_TRITON_DEFAULT_MODEL_NAME=sm_identity -SERVER_LOG="./sagemaker_max_input_size_server.log" -SERVER_ARGS="--allow-sagemaker=true --allow-http=true \ - --allow-grpc=false --allow-metrics=false \ - --model-repository=`pwd`/models_identity \ - --http-max-input-size=128" -run_server_nowait -sagemaker_wait_for_server_ready $SERVER_PID 10 -if [ "$WAIT_RET" != "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - kill $SERVER_PID || true - cat $SERVER_LOG - exit 1 -fi - -set +e - -# Small payload under 128 bytes should succeed -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8080/invocations \ - -H "Content-Type: application/json" \ - -d '{"inputs":[{"name":"INPUT0","datatype":"FP32","shape":[1,1],"data":[1.0]}],"outputs":[{"name":"OUTPUT0"}]}'` -if [ "$code" != "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected small payload to succeed on SageMaker endpoint (got $code)\n***" - RET=1 -fi - -# Large payload over 128 bytes should be rejected -rm -f ./curl.out -LARGE_PAYLOAD='{"inputs":[{"name":"INPUT0","datatype":"FP32","shape":[1,16],"data":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0]}],"outputs":[{"name":"OUTPUT0"}]}' -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8080/invocations \ - -H "Content-Type: application/json" \ - -d "$LARGE_PAYLOAD"` -if [ "$code" == "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected oversized payload to be rejected on SageMaker endpoint\n***" - RET=1 -fi - -# Same limit should apply to core HTTP endpoint -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/models/sm_identity/infer \ - -H "Content-Type: application/json" \ - -d "$LARGE_PAYLOAD"` -if [ "$code" == "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected oversized payload to be rejected on core HTTP endpoint\n***" - RET=1 -fi - -set -e - -unset SAGEMAKER_TRITON_DEFAULT_MODEL_NAME - -kill $SERVER_PID -wait $SERVER_PID - unlink /opt/ml/model rm -rf /opt/ml/model -rm -rf models_identity if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" diff --git a/qa/L0_sdk/test.sh b/qa/L0_sdk/test.sh index 84c6f429fb..20baf31639 100755 --- a/qa/L0_sdk/test.sh +++ b/qa/L0_sdk/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,13 +34,17 @@ set +e RET=0 -# Check image_client and perf_analyzer +# Check image_client and perf_client if [[ ! -x "triton_client/bin/image_client" ]]; then echo -e "*** image_client executable not present\n" RET=1 fi -if ! command -v perf_analyzer >/dev/null 2>&1; then - echo -e "*** perf_analyzer is not installed\n" +if [[ ! -x "triton_client/bin/perf_analyzer" ]]; then + echo -e "*** perf_analyzer executable is not present\n" + RET=1 +fi +if [[ ! -x "triton_client/bin/perf_client" ]]; then + echo -e "*** perf_client link is not present\n" RET=1 fi @@ -153,9 +157,11 @@ fi # we need to replace the text here as well to match the normalized version. WHLVERSION=`cat /workspace/TRITON_VERSION | sed 's/dev/\.dev0/'` if [[ "aarch64" != $(uname -m) ]] ; then - WHLS="tritonclient-${WHLVERSION}-py3-none-any.whl" + WHLS="tritonclient-${WHLVERSION}-py3-none-any.whl \ + tritonclient-${WHLVERSION}-py3-none-manylinux1_x86_64.whl" else - WHLS="tritonclient-${WHLVERSION}-py3-none-any.whl" + WHLS="tritonclient-${WHLVERSION}-py3-none-any.whl \ + tritonclient-${WHLVERSION}-py3-none-manylinux2014_aarch64.whl" fi for l in $WHLS; do if [[ ! -f "triton_client/python/$l" ]]; then @@ -173,7 +179,7 @@ python -c """import tritonclient; import tritonclient.grpc; import tritonclient. import tritonclient.utils.cuda_shared_memory; import tritonclient.utils.shared_memory""" RET=$(($RET+$?)) -EXECUTABLES="perf_analyzer" +EXECUTABLES="perf_analyzer perf_client" for l in $EXECUTABLES; do if [ $(which -a $l | grep "/usr/local/bin/$l" | wc -l) -ne 1 ]; then which -a $l diff --git a/qa/L0_sequence_batcher/test.sh b/qa/L0_sequence_batcher/test.sh index 5df80b9b90..229a935319 100755 --- a/qa/L0_sequence_batcher/test.sh +++ b/qa/L0_sequence_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -115,9 +115,6 @@ else fi SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR} --log-verbose=1" -if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then - SERVER_ARGS_EXTRA="${SERVER_ARGS_EXTRA} --allow-client-shm=true" -fi source ../common/util.sh diff --git a/qa/L0_sequence_stress/test.sh b/qa/L0_sequence_stress/test.sh index cb6e0328c8..b2bc66f8ac 100755 --- a/qa/L0_sequence_stress/test.sh +++ b/qa/L0_sequence_stress/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -85,45 +85,6 @@ for model_trial in 1 2 4 ; do wait $SERVER_PID done -# Test invalid gRPC infer handler thread count -for thread_cnt in -1 0 1 129; do - MODEL_DIR=models1 - SERVER_ARGS="--model-repository=`pwd`/$MODEL_DIR --grpc-infer-thread-count=$thread_cnt" - SERVER_LOG="./$MODEL_DIR.server.log" - run_server - if [ "$SERVER_PID" != "0" ]; then - echo -e "\n***\n*** Failed: $SERVER started successfully when it was expected to fail\n***" - RET=1 - kill SERVER_PID - wait $SERVER_PID - fi -done - -# Test gRPC infer handler thread count under stress -thread_cnt=128 -for model_trial in 1 2 4 ; do - MODEL_DIR=models${model_trial} - SERVER_ARGS="--model-repository=`pwd`/$MODEL_DIR --grpc-infer-thread-count=$thread_cnt" - SERVER_LOG="./$MODEL_DIR.server.log" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - python $STRESS_TEST >>$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - set -e - - kill $SERVER_PID - wait $SERVER_PID -done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else diff --git a/qa/L0_shared_memory/shared_memory_test.py b/qa/L0_shared_memory/shared_memory_test.py index 51bf914c88..8f3c2fbb52 100755 --- a/qa/L0_shared_memory/shared_memory_test.py +++ b/qa/L0_shared_memory/shared_memory_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ class SystemSharedMemoryTestBase(tu.TestResultCollector): DEFAULT_SHM_BYTE_SIZE = 64 - SYS_PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") def setUp(self): self._setup_client() @@ -114,6 +113,10 @@ def _configure_server( shm_op1_handle, ] # Implicit assumption that input and output byte_sizes are 64 bytes for now + input0_data = np.arange(start=0, stop=16, dtype=np.int32) + input1_data = np.ones(shape=16, dtype=np.int32) + shm.set_shared_memory_region(shm_ip0_handle, [input0_data]) + shm.set_shared_memory_region(shm_ip1_handle, [input1_data]) self.triton_client.register_system_shared_memory( "input0_data", "/input0_data", register_byte_size, offset=register_offset ) @@ -126,16 +129,6 @@ def _configure_server( self.triton_client.register_system_shared_memory( "output1_data", "/output1_data", register_byte_size, offset=register_offset ) - - # Write data to shared memory regions - input0_data = np.arange(start=0, stop=16, dtype=np.int32) - input1_data = np.ones(shape=16, dtype=np.int32) - shm.set_shared_memory_region( - shm_ip0_handle, [input0_data], offset=register_offset - ) - shm.set_shared_memory_region( - shm_ip1_handle, [input1_data], offset=register_offset - ) self.shm_names = ["input0_data", "input1_data", "output0_data", "output1_data"] def _cleanup_shm_handles(self): @@ -145,39 +138,6 @@ def _cleanup_shm_handles(self): class SharedMemoryTest(SystemSharedMemoryTestBase): - def test_client_shm_disabled_by_default(self): - # When the server is started without --allow-client-shm, registration and - # unregistration are rejected but querying status remains allowed (empty). - shm_op0_handle = shm.create_shared_memory_region("dummy_data", "/dummy_data", 8) - self._shm_handles.append(shm_op0_handle) - - shm_status_before = self.triton_client.get_system_shared_memory_status() - if self.protocol == "http": - self.assertEqual(len(shm_status_before), 0) - else: - self.assertEqual(len(shm_status_before.regions), 0) - - with self.assertRaisesRegex( - utils.InferenceServerException, - "Client shared memory is disabled", - ): - self.triton_client.register_system_shared_memory( - "dummy_data", "/dummy_data", 8 - ) - - with self.assertRaisesRegex( - utils.InferenceServerException, - "Client shared memory is disabled", - ): - self.triton_client.unregister_system_shared_memory("dummy_data") - - shm_status_after = self.triton_client.get_system_shared_memory_status() - self.assertEqual( - shm_status_before, - shm_status_after, - "system shared memory status must be unchanged after failed register/unregister", - ) - def test_invalid_create_shm(self): with self.assertRaisesRegex( shm.SharedMemoryException, "unable to create the shared memory region" @@ -332,45 +292,6 @@ def test_too_big_shm(self): self._shm_handles.append(shm_ip2_handle) self._cleanup_shm_handles() - def test_large_shm_register_offset(self): - # Test for out of bounds read vulnerability when registering system shared memory with large offset - - platforms = ( - ["python", "onnx", "libtorch", "plan", "openvino"] - if os.environ.get("BACKENDS") is None - else os.environ.get("BACKENDS").split() - ) - for platform in platforms: - model_name = f"{platform}_int32_int32_int32" - - # Test for large offset - error_msg = [] - # Create a large shm size (page_size * 1024 is large enough to reproduce a segfault). - # Register offset at 1 page before the end of the shm region to give enough space for the input/output data. - create_byte_size = self.SYS_PAGE_SIZE * 1024 - register_offset = self.SYS_PAGE_SIZE * 1023 - self._configure_server( - create_byte_size=create_byte_size, - register_offset=register_offset, - ) - - iu.shm_basic_infer( - self, - self.triton_client, - self._shm_handles[0], - self._shm_handles[1], - self._shm_handles[2], - self._shm_handles[3], - error_msg, - register_offset=register_offset, - protocol=self.protocol, - use_system_shared_memory=True, - override_model_name=model_name, - ) - self.triton_client.unregister_system_shared_memory() - if len(error_msg) > 0: - raise Exception(str(error_msg)) - def test_mixed_raw_shm(self): # Mix of shared memory and RAW inputs error_msg = [] @@ -411,12 +332,7 @@ def test_unregisterall(self): def test_infer_offset_out_of_bound(self): # Shared memory offset outside output region - Throws error error_msg = [] - create_byte_size = self.SYS_PAGE_SIZE + self.DEFAULT_SHM_BYTE_SIZE - register_offset = self.SYS_PAGE_SIZE - self._configure_server( - create_byte_size=create_byte_size, - register_offset=register_offset, - ) + self._configure_server() if self.protocol == "http": # -32 when placed in an int64 signed type, to get a negative offset # by overflowing @@ -446,13 +362,8 @@ def test_infer_offset_out_of_bound(self): def test_infer_byte_size_out_of_bound(self): # Shared memory byte_size outside output region - Throws error error_msg = [] - create_byte_size = self.SYS_PAGE_SIZE + self.DEFAULT_SHM_BYTE_SIZE - register_offset = self.SYS_PAGE_SIZE - self._configure_server( - create_byte_size=create_byte_size, - register_offset=register_offset, - ) - offset = 1 + self._configure_server() + offset = 60 byte_size = self.DEFAULT_SHM_BYTE_SIZE iu.shm_basic_infer( @@ -474,59 +385,6 @@ def test_infer_byte_size_out_of_bound(self): ) self._cleanup_shm_handles() - def test_infer_integer_overflow(self): - # Test for integer overflow vulnerability in offset + byte_size calculation - error_msg = [] - self._configure_server() - - offset = 32 - byte_size = 2**64 - 32 - - if self.protocol == "http": - iu.shm_basic_infer( - self, - self.triton_client, - self._shm_handles[0], - self._shm_handles[1], - self._shm_handles[2], - self._shm_handles[3], - error_msg, - shm_output_offset=offset, - shm_output_byte_size=byte_size, - protocol=self.protocol, - use_system_shared_memory=True, - ) - - self.assertEqual(len(error_msg), 1) - self.assertTrue( - "Integer overflow detected: byte_size " in error_msg[0], - f"Unexpected error message: {error_msg[0]}", - ) - self._cleanup_shm_handles() - else: - # The gRPC client utilizes the int64_param and will throw a separate error for values larger than 2**63-1 - try: - iu.shm_basic_infer( - self, - self.triton_client, - self._shm_handles[0], - self._shm_handles[1], - self._shm_handles[2], - self._shm_handles[3], - error_msg, - shm_output_offset=offset, - shm_output_byte_size=byte_size, - protocol=self.protocol, - use_system_shared_memory=True, - ) - self.assertTrue( - False, - "Expected gRPC client to fail on value larger than int64_param maximum", - ) - except ValueError as ex: - self.assertIn("Value out of range:", str(ex)) - self._cleanup_shm_handles() - def test_register_out_of_bound(self): create_byte_size = self.DEFAULT_SHM_BYTE_SIZE @@ -598,53 +456,6 @@ def test_python_client_leak(self): "client memory usage is increasing", ) - def test_register_reserved_names(self): - """ - Test that registration fails if attempting to use a reserved - prefix for the shm key. - """ - # This matches kTritonSharedMemoryRegionPrefix in the server code. - reserved_prefix = "triton_python_backend_shm_region_" - shm_name = "my_test_shm_name" - - # The shared memory key cannot start with the reserved prefix, - # regardless of leading slashes. - shm_keys_to_test = [ - f"{reserved_prefix}_my_test_shm_key", - f"/{reserved_prefix}_my_test_shm_key", - f"///{reserved_prefix}_my_test_shm_key", - ] - - for shm_key in shm_keys_to_test: - with self.subTest(shm_key=shm_key): - expected_msg = f"cannot register shared memory region '{shm_name}' with key '{shm_key}' as the key contains the reserved prefix '{reserved_prefix}'" - with self.assertRaisesRegex( - utils.InferenceServerException, expected_msg - ): - self.triton_client.register_system_shared_memory( - shm_name, shm_key, 10000 - ) - - def test_register_invalid_shm_key(self): - """ - Test that registration fails if attempting to use an invalid name for the shm key. - """ - shm_name = "my_test_shm_name" - shm_keys_to_test = [ - "/", - "///", - ] - - for shm_key in shm_keys_to_test: - with self.subTest(shm_key=shm_key): - expected_msg = f"cannot register shared memory region '{shm_name}' - invalid shm key '{shm_key}'" - with self.assertRaisesRegex( - utils.InferenceServerException, expected_msg - ): - self.triton_client.register_system_shared_memory( - shm_name, shm_key, 10000 - ) - def callback(user_data, result, error): if error: diff --git a/qa/L0_shared_memory/test.sh b/qa/L0_shared_memory/test.sh index b8a02338fc..27e8b2dd80 100755 --- a/qa/L0_shared_memory/test.sh +++ b/qa/L0_shared_memory/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,78 +25,21 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - CLIENT_LOG="./client.log" SHM_TEST=shared_memory_test.py TEST_RESULT_FILE='test_results.txt' # Configure to support test on jetson as well TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} -DATADIR=/data/inferenceserver/${REPO_VERSION} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends +SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR}" source ../common/util.sh pip3 install psutil RET=0 rm -fr *.log -# Test that shared memory registration/unregistration is rejected and status -# query is allowed when --allow-client-shm is not set (default is false). -SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR}" -for client_type in http grpc; do - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 ${SERVER_ARGS_EXTRA}" - SERVER_LOG="./test_client_shm_disabled_by_default.$client_type.server.log" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - export CLIENT_TYPE=$client_type - TMP_CLIENT_LOG="./tmp_client.log" - echo "Test: test_client_shm_disabled_by_default, client type: $client_type" >>$TMP_CLIENT_LOG - - set +e - python3 $SHM_TEST SharedMemoryTest.test_client_shm_disabled_by_default >>$TMP_CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - cat $TMP_CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $TEST_RESULT_FILE - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi - fi - cat $TMP_CLIENT_LOG >>$CLIENT_LOG - rm $TMP_CLIENT_LOG - kill $SERVER_PID - wait $SERVER_PID - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Server shut down non-gracefully\n***" - RET=1 - fi - set -e -done - -# Test shared memory registration with --allow-client-shm=true -SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR} --allow-client-shm=true" for i in \ test_invalid_create_shm \ test_valid_create_set_register \ @@ -110,10 +53,7 @@ for i in \ test_unregisterall \ test_infer_offset_out_of_bound \ test_infer_byte_size_out_of_bound \ - test_infer_integer_overflow \ test_register_out_of_bound \ - test_register_reserved_names \ - test_register_invalid_shm_key \ test_python_client_leak; do for client_type in http grpc; do SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 ${SERVER_ARGS_EXTRA}" @@ -200,60 +140,6 @@ for test_case in \ done done -# Test large system shared memory offset -rm -rf models/* -# prepare add_sub model of various backends -BACKENDS=${BACKENDS:-"python onnx libtorch plan openvino"} -for backend in ${BACKENDS} ; do - model="${backend}_int32_int32_int32" - model_dir="models/${model}" - if [[ $backend == "python" ]]; then - mkdir -p ${model_dir}/1 - cp ../python_models/add_sub/model.py ${model_dir}/1/ - cp ../python_models/add_sub/config.pbtxt ${model_dir}/ - sed -i 's/TYPE_FP32/TYPE_INT32/g' ${model_dir}/config.pbtxt - echo "max_batch_size: 8" >> ${model_dir}/config.pbtxt - else - mkdir -p ${model_dir} - cp -r $DATADIR/qa_model_repository/${model}/1 ${model_dir}/1 - cp $DATADIR/qa_model_repository/${model}/config.pbtxt ${model_dir}/ - cp $DATADIR/qa_model_repository/${model}/output0_labels.txt ${model_dir}/ - if [ $backend == "openvino" ]; then - echo 'parameters { key: "ENABLE_BATCH_PADDING" value { string_value: "YES" } }' >> models/${model}/config.pbtxt - fi - fi -done - -test_case="test_large_shm_register_offset" -for client_type in http grpc; do - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 ${SERVER_ARGS_EXTRA}" - SERVER_LOG="./${test_case}.${client_type}.server.log" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - export CLIENT_TYPE=$client_type - CLIENT_LOG="./${test_case}.${client_type}.client.log" - set +e - python3 $SHM_TEST SharedMemoryTest.${test_case} >>"$CLIENT_LOG" 2>&1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed - ${client_type}\n***" - RET=1 - fi - - kill $SERVER_PID - wait $SERVER_PID - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Server shut down non-gracefully\n***" - RET=1 - fi - set -e -done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else diff --git a/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/1/model.py b/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/1/model.py deleted file mode 100644 index 9b2a0a0141..0000000000 --- a/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/1/model.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import time - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Decoupled model that produces N responses based on input value. - """ - - def execute(self, requests): - for request in requests: - # Get input - number of responses to produce - in_tensor = pb_utils.get_input_tensor_by_name(request, "IN") - count = in_tensor.as_numpy().item() - - response_sender = request.get_response_sender() - out_tensor = pb_utils.Tensor("OUT", np.array([[0.5]], dtype=np.float32)) - - # Produce 'count' responses, each with 0.5 as the output value - for i in range(count): - time.sleep(0.1) # Simulate some processing delay - response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) - response_sender.send(response) - - # Send final flag - response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) - - return None diff --git a/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/config.pbtxt b/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/config.pbtxt deleted file mode 100644 index 0f7e05db0e..0000000000 --- a/qa/L0_simple_ensemble/backpressure_test_models/decoupled_producer/config.pbtxt +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -name: "decoupled_producer" -backend: "python" -max_batch_size: 1 - -input [ - { - name: "IN" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] - -output [ - { - name: "OUT" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] - -instance_group [ - { - count: 1 - kind: KIND_CPU - } -] - -model_transaction_policy { - decoupled: true -} diff --git a/qa/L0_simple_ensemble/backpressure_test_models/ensemble_disabled_max_inflight_requests/config.pbtxt b/qa/L0_simple_ensemble/backpressure_test_models/ensemble_disabled_max_inflight_requests/config.pbtxt deleted file mode 100644 index 804299cb21..0000000000 --- a/qa/L0_simple_ensemble/backpressure_test_models/ensemble_disabled_max_inflight_requests/config.pbtxt +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -platform: "ensemble" -max_batch_size: 0 - -input [ - { - name: "IN" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] - -output [ - { - name: "OUT" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] - -ensemble_scheduling { - step [ - { - model_name: "decoupled_producer" - model_version: -1 - input_map { - key: "IN" - value: "IN" - } - output_map { - key: "OUT" - value: "intermediate" - } - }, - { - model_name: "slow_consumer" - model_version: -1 - input_map { - key: "INPUT0" - value: "intermediate" - } - output_map { - key: "OUTPUT0" - value: "OUT" - } - } - ] -} - diff --git a/qa/L0_simple_ensemble/ensemble_backpressure_test.py b/qa/L0_simple_ensemble/ensemble_backpressure_test.py deleted file mode 100755 index e57b5b8cc0..0000000000 --- a/qa/L0_simple_ensemble/ensemble_backpressure_test.py +++ /dev/null @@ -1,554 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import os -import queue -import threading -import time -import unittest -from contextlib import ExitStack -from functools import partial - -import numpy as np -import test_util as tu -import tritonclient.grpc as grpcclient -from tritonclient.utils import InferenceServerException - -SERVER_URL = "localhost:8001" -DEFAULT_RESPONSE_TIMEOUT = 60 -EXPECTED_INFER_OUTPUT = 0.5 -MODEL_ENSEMBLE_PARALLEL_FAILED_ENQUEUE = "ensemble_parallel_step_failed_enqueue" -EXPECTED_PARALLEL_FAILED_ENQUEUE_OUTPUT = 4.0 - -NUM_REQUESTS = 16 -NUM_RESPONSES_PER_REQUEST = 8 - - -class UserData: - def __init__(self): - self._response_queue = queue.Queue() - - -def callback(user_data, result, error): - if error: - user_data._response_queue.put(error) - else: - user_data._response_queue.put(result) - - -def prepare_infer_args(input_value, enable_batching=False): - """ - Create InferInput/InferRequestedOutput lists - """ - if enable_batching: - input_data = np.array([[input_value]], dtype=np.int32) - else: - input_data = np.array([input_value], dtype=np.int32) - infer_input = [grpcclient.InferInput("IN", input_data.shape, "INT32")] - infer_input[0].set_data_from_numpy(input_data) - outputs = [grpcclient.InferRequestedOutput("OUT")] - return infer_input, outputs - - -def collect_responses(user_data, timeout=DEFAULT_RESPONSE_TIMEOUT): - """ - Collect responses from user_data until the final response flag is seen. - """ - errors = [] - responses = [] - while True: - try: - result = user_data._response_queue.get(timeout=timeout) - except queue.Empty: - raise Exception(f"No response received within {timeout} seconds.") - - if isinstance(result, InferenceServerException): - errors.append(result) - # error responses are final - stream terminates - break - - response = result.get_response() - # Add response to list if it has data (not empty final-only response) - if len(response.outputs) > 0: - responses.append(result) - - # Check if this is the final response - final = response.parameters.get("triton_final_response") - if final and final.bool_param: - break - - return errors, responses - - -class EnsembleBackpressureTest(tu.TestResultCollector): - """ - Tests for ensemble backpressure feature (max_inflight_requests). - """ - - def _run_inference( - self, model_name, expected_responses_per_request, num_concurrent_requests=1 - ): - """ - Send num_concurrent_requests streaming requests to model_name, each expecting - expected_responses_per_request responses. Verify all complete with correct data. - """ - user_datas = [UserData() for _ in range(num_concurrent_requests)] - - with ExitStack() as stack: - clients = [ - stack.enter_context(grpcclient.InferenceServerClient(SERVER_URL)) - for _ in range(num_concurrent_requests) - ] - - inputs, outputs = prepare_infer_args(expected_responses_per_request, True) - - # Start all concurrent requests - for i in range(num_concurrent_requests): - clients[i].start_stream(callback=partial(callback, user_datas[i])) - clients[i].async_stream_infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - - # Collect and verify responses for all requests - for i, ud in enumerate(user_datas): - errors, responses = collect_responses(ud) - self.assertEqual( - len(responses), - expected_responses_per_request, - f"Request {i}: expected {expected_responses_per_request} responses, got {len(responses)}", - ) - self.assertEqual( - len(errors), 0, f"Request {i}: unexpected errors: {errors}" - ) - # Verify correctness of responses - for idx, resp in enumerate(responses): - output = resp.as_numpy("OUT") - # output shape is [batch_size, 1]; extract scalar for comparison. - value = float(output[0][0]) - self.assertAlmostEqual( - value, - EXPECTED_INFER_OUTPUT, - places=5, - msg=f"Request {i} response {idx}: expected " - f"{EXPECTED_INFER_OUTPUT}, got {value}", - ) - - # Stop all streams - for client in clients: - client.stop_stream() - - def test_single_request_with_different_limits(self): - """ - Single streaming request that produces 16 responses via a three-step ensemble pipeline - (decoupled_producer → consumer_high_delay → consumer_low_delay) under various - max_inflight_requests configurations. - """ - cases = [ - ("ensemble_limit_4", "max_inflight_requests=4"), - ("ensemble_limit_1", "max_inflight_requests=1"), - ("ensemble_disabled", "max_inflight_requests is disabled"), - ] - for model_name, desc in cases: - with self.subTest(limit=desc): - self._run_inference( - model_name=model_name, expected_responses_per_request=16 - ) - - def test_concurrent_requests_with_different_limits(self): - """ - NUM_REQUESTS concurrent streaming requests (NUM_RESPONSES_PER_REQUEST - responses each) exercise the max_inflight_requests limit. - Subtests cover: limit=4, limit=1, and the limit disabled. - """ - cases = [ - ("ensemble_limit_4", "max_inflight_requests=4"), - ("ensemble_limit_1", "max_inflight_requests=1"), - ("ensemble_disabled", "max_inflight_requests is disabled"), - ] - for model_name, desc in cases: - with self.subTest(limit=desc): - self._run_inference( - model_name=model_name, - expected_responses_per_request=NUM_RESPONSES_PER_REQUEST, - num_concurrent_requests=NUM_REQUESTS, - ) - - def test_sequential_requests_limiter_resets_cleanly(self): - """ - Send NUM_REQUESTS requests one after another. If the limiter - leaks a slot on any request, subsequent requests will be stuck or time out. - """ - for seq_idx in range(NUM_REQUESTS): - with self.subTest(request=seq_idx): - self._run_inference( - model_name="ensemble_limit_4", - expected_responses_per_request=NUM_RESPONSES_PER_REQUEST, - ) - - def test_request_cancellation_under_backpressure(self): - """ - Start a long-running request (32 responses), cancel mid-stream, - and verify the server sends a CANCELLED status and only a partial set of - responses is received. - """ - input_value = 32 - user_data = UserData() - - with grpcclient.InferenceServerClient(SERVER_URL) as triton_client: - inputs, outputs = prepare_infer_args(input_value, True) - triton_client.start_stream(callback=partial(callback, user_data)) - - # Start the request - triton_client.async_stream_infer( - model_name="ensemble_limit_4", inputs=inputs, outputs=outputs - ) - - responses = [] - try: - result = user_data._response_queue.get(timeout=5) - if isinstance(result, InferenceServerException): - self.fail(f"Got error before cancellation: {result}") - resp = result.get_response() - if len(resp.outputs) > 0: - responses.append(result) - except queue.Empty: - self.fail("Stream did not produce any response before cancellation.") - - # Cancel the stream - this unblocks any waiting producers and triggers a CANCELLED error. - triton_client.stop_stream(cancel_requests=True) - - # Allow some time for cancellation - time.sleep(1) - - cancellation_found = False - while True: - try: - result = user_data._response_queue.get(timeout=1) - if isinstance(result, InferenceServerException): - self.assertEqual( - result.status(), - "StatusCode.CANCELLED", - f"Expected CANCELLED status, got: {result.status()}", - ) - cancellation_found = True - break - else: - response = result.get_response() - if len(response.outputs) > 0: - responses.append(result) - # Check for final response - final = response.parameters.get("triton_final_response") - if final and final.bool_param: - break - except queue.Empty: - break - - # Verify the cancellation error was received - self.assertTrue( - cancellation_found, - "Did not receive the expected cancellation error from the server.", - ) - - # Verify we received only a partial set of responses - self.assertLess( - len(responses), - input_value, - "Expected partial responses due to cancellation, but received all of them.", - ) - self.assertGreater( - len(responses), - 0, - "Expected to receive at least one response before cancellation.", - ) - - -class EnsembleStepMaxQueueSizeTest(tu.TestResultCollector): - def _run_inference(self, model_name, expected_responses_count): - """ - Helper function for streaming inference. - - For decoupled streaming ensembles with queue limit on internal step: - - Each producer response creates an independent flow through the ensemble - - Flows that complete before error is set send their outputs successfully - - Once error occurs (queue full), stream terminates with error - - Result: 0-N successful responses + 1 error (N depends on timing) - """ - user_data = UserData() - with grpcclient.InferenceServerClient(SERVER_URL) as triton_client: - try: - inputs, outputs = prepare_infer_args(expected_responses_count) - triton_client.start_stream(callback=partial(callback, user_data)) - triton_client.async_stream_infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - - # Collect and verify responses - errors, responses = collect_responses(user_data) - self.assertGreaterEqual( - len(responses), - 0, - "May have 0 or more successful responses depending on timing", - ) - self.assertLess( - len(responses), - expected_responses_count, - f"Should have fewer than {expected_responses_count} responses (some flows failed)", - ) - self.assertEqual( - len(errors), - 1, - "Expected exactly one error when the queue is full and the stream terminates", - ) - - # Verify correctness of successful responses - for idx, resp in enumerate(responses): - output = resp.as_numpy("OUT") - self.assertAlmostEqual( - output[0], - EXPECTED_INFER_OUTPUT, - places=5, - msg=f"Response {idx} has incorrect value - {output[0]}", - ) - - # Verify error is queue-full error - self.assertIn( - "Exceeds maximum queue size", - str(errors[0]), - f"Expected queue size error, got: {str(errors[0])}", - ) - finally: - triton_client.stop_stream() - - def _run_concurrent_inference(self, model_name, expected_responses_count): - """ - Helper function for concurrent independent requests. - Each request either succeeds completely or fails completely. - Returns: (num_successes, num_errors) tuple - """ - user_data = UserData() - with grpcclient.InferenceServerClient(SERVER_URL) as triton_client: - try: - inputs, outputs = prepare_infer_args(expected_responses_count) - triton_client.start_stream(callback=partial(callback, user_data)) - triton_client.async_stream_infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - - # Collect responses - errors, responses = collect_responses(user_data) - - # For concurrent independent requests with queue limit on internal step: - # - Requests that arrive before queue fills: succeed with all outputs - # - Requests that arrive after queue fills: fail with error - total = len(responses) + len(errors) - self.assertEqual( - total, - expected_responses_count, - f"Expected {expected_responses_count} total responses, got {total}", - ) - - if len(errors) > 0: - # This request failed - self.assertEqual( - len(responses), - 0, - "Failed request should have no successful outputs", - ) - self.assertEqual( - len(errors), 1, "Failed request should have exactly one error" - ) - self.assertIn( - "Exceeds maximum queue size", - str(errors[0]), - f"Expected queue size error, got: {str(errors[0])}", - ) - return (0, 1) # 0 successes, 1 error - else: - # This request succeeded - self.assertEqual( - len(responses), - expected_responses_count, - f"Successful request should have all {expected_responses_count} outputs", - ) - # Verify correctness of successful responses - for idx, resp in enumerate(responses): - output = resp.as_numpy("OUT") - self.assertAlmostEqual( - output[0], - EXPECTED_INFER_OUTPUT, - places=5, - msg=f"Response {idx} has incorrect value - {output[0]}", - ) - return (expected_responses_count, 0) # N successes, 0 errors - finally: - triton_client.stop_stream() - - def test_step1_max_queue_size(self): - """ - Test max_queue_size on step 1 (decoupled_producer). - - Trigger 32 concurrent ensemble requests, each producing 1 response - - Step 1 (producer) has max_queue_size limit - - Some ensemble requests succeed completely (before queue fills) - - Some fail completely (when producer queue is full) - """ - model_name = "ensemble_step1_enabled_max_queue_size" - num_requests = 32 - - # Store results from each thread - results = [] - - def thread_wrapper(model_name, expected_count, results_list): - """Wrapper to capture thread results""" - result = self._run_concurrent_inference(model_name, expected_count) - results_list.append(result) - - # Launch concurrent threads to perform infer requests - threads = [] - for i in range(num_requests): - t = threading.Thread(target=thread_wrapper, args=(model_name, 1, results)) - threads.append(t) - t.start() - - # Wait for all requests to complete - for t in threads: - t.join(timeout=60) - - # Aggregate results from all threads - total_successes = sum(r[0] for r in results) - total_errors = sum(r[1] for r in results) - - # Verify aggregate behavior - self.assertEqual( - total_successes + total_errors, - num_requests, - f"Expected {num_requests} total results (successes + errors), " - f"got {total_successes} successes + {total_errors} errors = {total_successes + total_errors}", - ) - - # Verify at least some errors occurred (queue limit was hit) - self.assertGreater( - total_errors, - 0, - f"Expected some errors due to max_queue_size limit, " - f"but all {num_requests} requests succeeded.", - ) - - # Verify at least some successes occurred (not all rejected) - self.assertGreater( - total_successes, - 0, - f"Expected some successful requests before queue filled, " - f"but all {num_requests} requests failed.", - ) - - def test_step2_max_queue_size(self): - """ - Test max_queue_size on step 2 (slow_consumer). - - Trigger 1 streaming ensemble request producing 32 responses - - Step 1 (producer) generates 32 responses rapidly (every 100ms) - - Step 2 (consumer) has max_queue_size=5 and processes slowly (500ms each) - - Each producer response is an independent request to the second step through - - the ensemble flow. Some requests complete successfully before queue fills - - When queue fills, error is set and stream terminates - - All inflight steps drain, then error response sent to client - """ - model_name = "ensemble_step2_enabled_max_queue_size" - self._run_inference(model_name=model_name, expected_responses_count=32) - - -class EnsembleParallelFailedEnqueueTest(tu.TestResultCollector): - def _run_inference(self, expected_responses_count=32): - """ - Exercise a fan-out ensemble where one parallel branch hits queue-full - first. Successful responses emitted before the failure should still be - correct, and the stream should terminate with exactly one queue-full - error. - """ - user_data = UserData() - with grpcclient.InferenceServerClient(SERVER_URL) as triton_client: - try: - inputs, outputs = prepare_infer_args(expected_responses_count) - triton_client.start_stream(callback=partial(callback, user_data)) - triton_client.async_stream_infer( - model_name=MODEL_ENSEMBLE_PARALLEL_FAILED_ENQUEUE, - inputs=inputs, - outputs=outputs, - ) - - errors, responses = collect_responses(user_data, timeout=15) - self.assertLess( - len(responses), - expected_responses_count, - "Expected the parallel slow branch to queue-fill before all " - "responses completed.", - ) - self.assertEqual( - len(errors), - 1, - "Expected exactly one queue-full error from the parallel " - "failed-enqueue path.", - ) - self.assertIn( - "Exceeds maximum queue size", - str(errors[0]), - f"Expected queue size error, got: {str(errors[0])}", - ) - - for idx, resp in enumerate(responses): - output = resp.as_numpy("OUT") - self.assertAlmostEqual( - float(np.squeeze(output)), - EXPECTED_PARALLEL_FAILED_ENQUEUE_OUTPUT, - places=5, - msg=f"Response {idx} has incorrect value - {output}", - ) - finally: - triton_client.stop_stream() - - def test_parallel_step_failed_enqueue(self): - """ - Repeat the same request according to PARALLEL_FAILED_ENQUEUE_LOOPS. - """ - loop_count = int(os.environ.get("PARALLEL_FAILED_ENQUEUE_LOOPS", "1")) - self.assertGreaterEqual( - loop_count, 1, "PARALLEL_FAILED_ENQUEUE_LOOPS must be >= 1" - ) - - for iteration in range(loop_count): - with self.subTest(iteration=iteration): - self._run_inference() - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_simple_ensemble/test.sh b/qa/L0_simple_ensemble/test.sh index 1e62c91e7b..927f36ea32 100755 --- a/qa/L0_simple_ensemble/test.sh +++ b/qa/L0_simple_ensemble/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,17 +31,15 @@ SIMPLE_TEST_PY=./ensemble_test.py CLIENT_LOG="./client.log" -TEST_MODEL_DIR="`pwd`/models" -BACKPRESSURE_TEST_MODEL_DIR="`pwd`/backpressure_test_models" TEST_RESULT_FILE='test_results.txt' SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=${TEST_MODEL_DIR}" +SERVER_ARGS="--model-repository=`pwd`/models" SERVER_LOG="./inference_server.log" source ../common/util.sh # ensure ensemble models have version sub-directory -mkdir -p ${TEST_MODEL_DIR}/ensemble_add_sub_int32_int32_int32/1 -mkdir -p ${TEST_MODEL_DIR}/ensemble_partial_add_sub/1 +mkdir -p `pwd`/models/ensemble_add_sub_int32_int32_int32/1 +mkdir -p `pwd`/models/ensemble_partial_add_sub/1 rm -f $CLIENT_LOG $SERVER_LOG @@ -148,475 +146,10 @@ set -e kill $SERVER_PID wait $SERVER_PID -######## Test max_queue_size dynamic batching parameter in ensemble steps ######## -## Ensemble model: step1-decoupled_producer -> step2-slow_consumer -MAX_QUEUE_SIZE_TEST_MODEL_DIR="`pwd`/max_queue_size_test_models" -rm -rf ${MAX_QUEUE_SIZE_TEST_MODEL_DIR} - -# Enable max_queue_size in the first step (decoupled_producer) -mkdir -p ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step1_enabled_max_queue_size/1 \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer_enabled_max_queue_size/1 \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/ensemble_disabled_max_inflight_requests/config.pbtxt \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step1_enabled_max_queue_size/ -sed -i 's/"decoupled_producer"/"decoupled_producer_enabled_max_queue_size"/g' \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step1_enabled_max_queue_size/config.pbtxt - -cp ../python_models/ground_truth/model.py ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer/1 -cp ../python_models/ground_truth/config.pbtxt ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer/ -sed -i 's/name: "ground_truth"/name: "slow_consumer"/g' ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer/config.pbtxt -sed -i 's/max_batch_size: 64/max_batch_size: 1/g' ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer/config.pbtxt - -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1/model.py ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer_enabled_max_queue_size/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/config.pbtxt ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer_enabled_max_queue_size/ -sed -i 's/name: "decoupled_producer"/name: "decoupled_producer_enabled_max_queue_size"/g' ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer_enabled_max_queue_size/config.pbtxt -# Add dynamic_batching with max_queue_size to decoupled_producer -cat >> ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer_enabled_max_queue_size/config.pbtxt << 'EOF' - -dynamic_batching { - preferred_batch_size: [ 1 ] - default_queue_policy { - max_queue_size: 4 - } -} -EOF - -# Enable max_queue_size in the second step (slow_consumer) -mkdir -p ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step2_enabled_max_queue_size/1 \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer/1 ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/ensemble_disabled_max_inflight_requests/config.pbtxt \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step2_enabled_max_queue_size/ -sed -i 's/"slow_consumer"/"slow_consumer_enabled_max_queue_size"/g' \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/ensemble_step2_enabled_max_queue_size/config.pbtxt - -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1/model.py ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/config.pbtxt ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer/ - -cp ../python_models/ground_truth/model.py ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/1 -cp ../python_models/ground_truth/config.pbtxt ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/ -sed -i 's/name: "ground_truth"/name: "slow_consumer_enabled_max_queue_size"/g' \ - ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/config.pbtxt -sed -i 's/max_batch_size: 64/max_batch_size: 1/g' ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/config.pbtxt -# Add dynamic_batching with max_queue_size to slow_consumer -cat >> ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer_enabled_max_queue_size/config.pbtxt << 'EOF' - -dynamic_batching { - preferred_batch_size: [ 1 ] - default_queue_policy { - max_queue_size: 4 - } -} -EOF - -BACKPRESSURE_TEST_PY=./ensemble_backpressure_test.py -TEST_NAME="EnsembleStepMaxQueueSizeTest" -SERVER_LOG="./ensemble_step_max_queue_size_test_server.log" -CLIENT_LOG="./ensemble_step_max_queue_size_test_client.log" -rm -f $SERVER_LOG $CLIENT_LOG - -SERVER_ARGS="--model-repository=${MAX_QUEUE_SIZE_TEST_MODEL_DIR}" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python $BACKPRESSURE_TEST_PY $TEST_NAME -v >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - RET=1 - cat $CLIENT_LOG -else - check_test_results $TEST_RESULT_FILE 2 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - - -######## Test parallel-step failed enqueue path in ensemble scheduler ######## -PARALLEL_FAILED_ENQUEUE_MODEL_DIR="`pwd`/parallel_failed_enqueue_test_models" -rm -rf ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR} - -mkdir -p ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/ensemble_parallel_step_failed_enqueue/1 -mkdir -p ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/decoupled_producer_parallel_queue/1 -mkdir -p ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/1 -mkdir -p ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/1 -mkdir -p ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/join_add_sub/1 - -# Producer emits repeated responses with a larger payload value so the -# queue-limited branch fills first. -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1/model.py \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/decoupled_producer_parallel_queue/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/config.pbtxt \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/decoupled_producer_parallel_queue/ -sed -i 's/name: "decoupled_producer"/name: "decoupled_producer_parallel_queue"/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/decoupled_producer_parallel_queue/config.pbtxt -sed -i 's/0.5/2.0/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/decoupled_producer_parallel_queue/1/model.py - -# Queue-limited branch used to trigger a failed enqueue. -cp ../python_models/ground_truth/model.py \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/1 -cp ../python_models/ground_truth/config.pbtxt \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/ -sed -i 's/name: "ground_truth"/name: "slow_consumer_queue_limited"/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/config.pbtxt -sed -i 's/max_batch_size: 64/max_batch_size: 1/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/config.pbtxt -cat >> ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/slow_consumer_queue_limited/config.pbtxt << 'EOF' - -dynamic_batching { - preferred_batch_size: [ 1 ] - default_queue_policy { - max_queue_size: 1 - } -} -EOF - -# Parallel branch with the same interface and no added delay. -cp ../python_models/ground_truth/model.py ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/1 -cp ../python_models/ground_truth/config.pbtxt ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/ -sed -i 's/name: "ground_truth"/name: "fast_consumer"/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/config.pbtxt -sed -i 's/max_batch_size: 64/max_batch_size: 1/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/config.pbtxt -sed -i 's/time.sleep(delay)/time.sleep(0)/g' \ - ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/fast_consumer/1/model.py - -# Join both parallel branches into the ensemble output. -cp ../python_models/join_add_sub/model.py ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/join_add_sub/1 -cp ../python_models/join_add_sub/config.pbtxt ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/join_add_sub/ - -cat > ${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}/ensemble_parallel_step_failed_enqueue/config.pbtxt << 'EOF' -name: "ensemble_parallel_step_failed_enqueue" -platform: "ensemble" -max_batch_size: 0 - -input [ - { - name: "IN" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] - -output [ - { - name: "OUT" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] - -ensemble_scheduling { - step [ - { - model_name: "decoupled_producer_parallel_queue" - model_version: -1 - input_map { - key: "IN" - value: "IN" - } - output_map { - key: "OUT" - value: "intermediate" - } - }, - { - model_name: "slow_consumer_queue_limited" - model_version: -1 - input_map { - key: "INPUT0" - value: "intermediate" - } - output_map { - key: "OUTPUT0" - value: "slow_out" - } - }, - { - model_name: "fast_consumer" - model_version: -1 - input_map { - key: "INPUT0" - value: "intermediate" - } - output_map { - key: "OUTPUT0" - value: "fast_out" - } - }, - { - model_name: "join_add_sub" - model_version: -1 - input_map { - key: "INPUT0" - value: "slow_out" - } - input_map { - key: "INPUT1" - value: "fast_out" - } - output_map { - key: "OUTPUT0" - value: "OUT" - } - } - ] -} -EOF - -BACKPRESSURE_TEST_PY=./ensemble_backpressure_test.py -TEST_NAME="EnsembleParallelFailedEnqueueTest.test_parallel_step_failed_enqueue" -SERVER_LOG="./ensemble_parallel_failed_enqueue_test_server.log" -CLIENT_LOG="./ensemble_parallel_failed_enqueue_test_client.log" -rm -f $SERVER_LOG $CLIENT_LOG - -SERVER_ARGS="--model-repository=${PARALLEL_FAILED_ENQUEUE_MODEL_DIR}" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -PARALLEL_FAILED_ENQUEUE_LOOPS=${PARALLEL_FAILED_ENQUEUE_LOOPS:-1} \ -python $BACKPRESSURE_TEST_PY $TEST_NAME -v >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - RET=1 - cat $CLIENT_LOG -else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi - -if ! kill -0 $SERVER_PID > /dev/null 2>&1; then - cat $SERVER_LOG - echo -e "\n***\n*** Server exited during parallel failed enqueue test\n***" - RET=1 -else - wait_for_server_live $SERVER_PID 5 - if [ "$WAIT_RET" != "0" ]; then - cat $SERVER_LOG - echo -e "\n***\n*** Server did not remain live after parallel failed enqueue test\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID > /dev/null 2>&1 || true -wait $SERVER_PID > /dev/null 2>&1 || true - - -######## Test backpressure feature - 'max_inflight_requests' config option ######## -ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR="`pwd`/ensemble_backpressure_test_models" -rm -rf ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR} - -TEST_NAME="EnsembleBackpressureTest" -SERVER_LOG="./ensemble_backpressure_test_server.log" -CLIENT_LOG="./ensemble_backpressure_test_client.log" -SERVER_ARGS="--model-repository=${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}" -rm -f $SERVER_LOG $CLIENT_LOG - -# Step 1 - decoupled_producer (batch size 2) -mkdir -p ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1/model.py ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/1/ -cp ${BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/config.pbtxt ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/ -sed -i 's/max_batch_size: 1/max_batch_size: 2/g' ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/decoupled_producer/config.pbtxt - -generate_consumer_model() { - local name=$1 - local delay=$2 - - mkdir -p ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/${name}/1 - cat > ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/${name}/1/model.py << EOF -import time -import triton_python_backend_utils as pb_utils - -class TritonPythonModel: - def execute(self, requests): - responses = [] - for request in requests: - in_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", in_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - time.sleep(${delay}) - return responses -EOF - cat > ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/${name}/config.pbtxt << EOF -name: "${name}" -backend: "python" -max_batch_size: 2 -input [ { name: "INPUT0", data_type: TYPE_FP32, dims: [ 1 ] } ] -output [ { name: "OUTPUT0", data_type: TYPE_FP32, dims: [ 1 ] } ] -instance_group [ { count: 1, kind: KIND_CPU } ] -dynamic_batching { preferred_batch_size: [ 2 ] } -EOF -} - -generate_ensemble_model() { - local name=$1 - local limit=$2 - local batch_size=2 - - local limit_str="" - if [ "$limit" != "disabled" ]; then - limit_str="max_inflight_requests: $limit" - fi - - mkdir -p ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/${name}/1 - cat > ${ENSEMBLE_BACKPRESSURE_TEST_MODEL_DIR}/${name}/config.pbtxt << EOF -name: "${name}" -platform: "ensemble" -max_batch_size: ${batch_size} -input [ { name: "IN", data_type: TYPE_INT32, dims: [ 1 ] } ] -output [ { name: "OUT", data_type: TYPE_FP32, dims: [ 1 ] } ] -ensemble_scheduling { - ${limit_str} - step [ - { - model_name: "decoupled_producer" - model_version: -1 - input_map { key: "IN", value: "IN" } - output_map { key: "OUT", value: "intermediate_1" } - }, - { - model_name: "consumer_high_delay" - model_version: -1 - input_map { key: "INPUT0", value: "intermediate_1" } - output_map { key: "OUTPUT0", value: "intermediate_2" } - }, - { - model_name: "consumer_low_delay" - model_version: -1 - input_map { key: "INPUT0", value: "intermediate_2" } - output_map { key: "OUTPUT0", value: "OUT" } - } - ] -} -EOF -} - -# Steps 2 and 3 - consumer_high_delay and consumer_low_delay (batch size 2) -generate_consumer_model "consumer_high_delay" "0.5" -generate_consumer_model "consumer_low_delay" "0.1" - -# Ensemble models with different max_inflight_requests limits (including disabled) -generate_ensemble_model "ensemble_disabled" "disabled" -generate_ensemble_model "ensemble_limit_1" 1 -generate_ensemble_model "ensemble_limit_4" 4 - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e - -# Verify valid config was loaded successfully -if ! grep -q "Ensemble model 'ensemble_limit_1' configured with max_inflight_requests: 1" $SERVER_LOG; then - echo -e "\n***\n*** FAILED: ensemble_limit_1 did not load\n***" - RET=1 -fi -if ! grep -q "Ensemble model 'ensemble_limit_4' configured with max_inflight_requests: 4" $SERVER_LOG; then - echo -e "\n***\n*** FAILED: ensemble_limit_4 did not load\n***" - RET=1 -fi - -python $BACKPRESSURE_TEST_PY $TEST_NAME -v >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - RET=1 - cat $CLIENT_LOG -else - check_test_results $TEST_RESULT_FILE 4 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - - -######## Test invalid values for 'max_inflight_requests' config option ######## -INVALID_PARAM_MODEL_DIR="`pwd`/invalid_param_test_models" -SERVER_ARGS="--model-repository=${INVALID_PARAM_MODEL_DIR}" -SERVER_LOG="./invalid_max_inflight_requests_server.log" -rm -rf $SERVER_LOG ${INVALID_PARAM_MODEL_DIR} - -mkdir -p ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_negative_limit/1 -mkdir -p ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_string_limit/1 -mkdir -p ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_large_value_limit/1 -# Reuse the decoupled_producer and slow_consumer models built in the previous test section. -cp -r ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/decoupled_producer ${MAX_QUEUE_SIZE_TEST_MODEL_DIR}/slow_consumer ${INVALID_PARAM_MODEL_DIR}/ - -# max_inflight_requests = -5 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/ensemble_disabled_max_inflight_requests/config.pbtxt ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_negative_limit/ -sed -i 's/ensemble_scheduling {/ensemble_scheduling {\n max_inflight_requests: -5/g' \ - ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_negative_limit/config.pbtxt - -# max_inflight_requests = "invalid_value" -cp ${BACKPRESSURE_TEST_MODEL_DIR}/ensemble_disabled_max_inflight_requests/config.pbtxt ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_string_limit/ -sed -i 's/ensemble_scheduling {/ensemble_scheduling {\n max_inflight_requests: "invalid_value"/g' \ - ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_string_limit/config.pbtxt - -# max_inflight_requests = 12345678901 -cp ${BACKPRESSURE_TEST_MODEL_DIR}/ensemble_disabled_max_inflight_requests/config.pbtxt ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_large_value_limit/ -sed -i 's/ensemble_scheduling {/ensemble_scheduling {\n max_inflight_requests: 12345678901/g' \ - ${INVALID_PARAM_MODEL_DIR}/ensemble_invalid_large_value_limit/config.pbtxt - - -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "\n***\n*** FAILED: unexpected success starting $SERVER\n***" - kill $SERVER_PID - wait $SERVER_PID - cat $SERVER_LOG - RET=1 -fi - -set +e -# Verify negative value caused model load failure -if ! grep -q "Expected integer, got: -" $SERVER_LOG; then - echo -e "\n***\n*** FAILED: Negative value should fail model load\n***" - RET=1 -fi - -# Verify invalid string caused model load failure -if ! grep -q 'Expected integer, got: "invalid_value"' $SERVER_LOG; then - echo -e "\n***\n*** FAILED: Invalid string should fail model load\n***" - RET=1 -fi - -# Verify very large value caused model load failure -if ! grep -q "Integer out of range (12345678901)" $SERVER_LOG; then - echo -e "\n***\n*** FAILED: Large value should fail model load\n***" - RET=1 -fi -set -e - - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else + cat $CLIENT_LOG echo -e "\n***\n*** Test FAILED\n***" fi diff --git a/qa/L0_storage_azure/test.sh b/qa/L0_storage_azure/test.sh index e2d9de760c..805c081679 100755 --- a/qa/L0_storage_azure/test.sh +++ b/qa/L0_storage_azure/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -137,10 +137,7 @@ sleep 10 # Test 1 Scenarios: # 1. access blob using shared key in envs -# 2. access blob using system-assigned managed identity -# 3. access blob using user-assigned managed identity -# 4. access blob using DefaultAzureCredential -# 5. adding more scenarios in future +# 2. adding more scenarios in future for ENV_VAR in "shared_key"; do SERVER_LOG=$SERVER_LOG_BASE.$ENV_VAR.log CLIENT_LOG=$CLIENT_LOG_BASE.$ENV_VAR.log @@ -172,117 +169,6 @@ for ENV_VAR in "shared_key"; do wait $SERVER_PID done -# Test 2: Managed Identity authentication -# Requires the test host (VM/AKS) to have a system-assigned managed identity -# with Storage Blob Data Reader on the test storage account. -# Skip if not running in an MI-capable environment. -if [ ! -z "$TEST_AZURE_MANAGED_IDENTITY" ]; then - echo -e "\n***\n*** Testing system-assigned Managed Identity\n***" - - # Save original key and clear it so it won't be used - SAVED_AZURE_STORAGE_KEY=$AZURE_STORAGE_KEY - unset AZURE_STORAGE_KEY - export AZURE_STORAGE_AUTH_TYPE="managed_identity" - - SERVER_LOG=$SERVER_LOG_BASE.managed_identity_system.log - CLIENT_LOG=$CLIENT_LOG_BASE.managed_identity_system.log - MODEL_REPO="${AS_URL}/models" - SERVER_ARGS="--model-repository=$MODEL_REPO --exit-timeout-secs=120" - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER with system-assigned MI\n***" - cat $SERVER_LOG - RET=1 - else - set +e - run_unit_tests - set -e - - kill $SERVER_PID - wait $SERVER_PID - fi - - # Test 3: User-assigned Managed Identity (if client ID is provided) - if [ ! -z "$AZURE_STORAGE_CLIENT_ID" ]; then - echo -e "\n***\n*** Testing user-assigned Managed Identity\n***" - - SERVER_LOG=$SERVER_LOG_BASE.managed_identity_user.log - CLIENT_LOG=$CLIENT_LOG_BASE.managed_identity_user.log - SERVER_ARGS="--model-repository=$MODEL_REPO --exit-timeout-secs=120" - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER with user-assigned MI\n***" - cat $SERVER_LOG - RET=1 - else - set +e - run_unit_tests - set -e - - kill $SERVER_PID - wait $SERVER_PID - fi - else - echo -e "\n***\n*** Skipping user-assigned MI test (AZURE_STORAGE_CLIENT_ID not set)\n***" - fi - - # Test 4: DefaultAzureCredential chain - echo -e "\n***\n*** Testing DefaultAzureCredential\n***" - export AZURE_STORAGE_AUTH_TYPE="default" - unset AZURE_STORAGE_CLIENT_ID - - SERVER_LOG=$SERVER_LOG_BASE.default_credential.log - CLIENT_LOG=$CLIENT_LOG_BASE.default_credential.log - SERVER_ARGS="--model-repository=$MODEL_REPO --exit-timeout-secs=120" - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER with DefaultAzureCredential\n***" - cat $SERVER_LOG - RET=1 - else - set +e - run_unit_tests - set -e - - kill $SERVER_PID - wait $SERVER_PID - fi - - # Test: invalid auth_type should fail gracefully - echo -e "\n***\n*** Testing invalid auth_type (expect failure)\n***" - export AZURE_STORAGE_AUTH_TYPE="invalid_type" - - SERVER_LOG=$SERVER_LOG_BASE.invalid_auth_type.log - SERVER_ARGS="--model-repository=$MODEL_REPO --exit-timeout-secs=120 --exit-on-error=false" - - run_server - if [ "$SERVER_PID" != "0" ]; then - # Server started — but model load should have failed. Verify the log - # contains an authentication error rather than a successful load. - if grep -q "Unable to create Azure filesystem client" $SERVER_LOG; then - echo -e "*** invalid auth_type correctly rejected ***" - else - echo -e "\n***\n*** Expected auth failure with invalid auth_type\n***" - cat $SERVER_LOG - RET=1 - fi - kill $SERVER_PID - wait $SERVER_PID - else - echo -e "*** Server correctly refused to start with invalid auth_type ***" - fi - - # Restore environment for remaining tests - unset AZURE_STORAGE_AUTH_TYPE - unset AZURE_STORAGE_CLIENT_ID - export AZURE_STORAGE_KEY=$SAVED_AZURE_STORAGE_KEY -else - echo -e "\n***\n*** Skipping Managed Identity tests (TEST_AZURE_MANAGED_IDENTITY not set)\n***" -fi - # Test localization to a specified location export TRITON_AZURE_MOUNT_DIRECTORY=`pwd`/azure_localization_test diff --git a/qa/L0_storage_swiftstack/infer_test.py b/qa/L0_storage_swiftstack/infer_test.py index a263991ea1..922caeb900 100755 --- a/qa/L0_storage_swiftstack/infer_test.py +++ b/qa/L0_storage_swiftstack/infer_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -87,6 +87,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size, 1, 1), + (input_size, 1, 1), + (input_size, 1, 1), ): if input_dtype == np.int8: _infer_exact_helper( @@ -119,6 +122,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size,), + (input_size,), + (input_size,), ): _infer_exact_helper( self, diff --git a/qa/L0_torch_aoti/test.sh b/qa/L0_torch_aoti/test.sh deleted file mode 100755 index f37751c55e..0000000000 --- a/qa/L0_torch_aoti/test.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -source ../common/util.sh - -if [[ "${DEBUG}" == "true" ]]; then - set -x -else - set +x -fi - -COLOR_DARK="\033[90m" -COLOR_ERROR="\033[31m" -COLOR_INFO="\033[94m" -COLOR_RESET="\033[0m" -COLOR_STATUS="\033[36m" -COLOR_SUCCESS="\033[32m" -COLOR_WARNING="\033[33m" -RET=0 - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [[ "$#" -ge 1 ]]; then - REPO_VERSION=$1 -fi -if [[ -z "$REPO_VERSION" ]]; then - echo -e "${COLOR_ERROR}Repository version must be specified${COLOR_RESET}" 1>&2 - echo -e "${COLOR_ERROR}\n***\n*** Test Failed\n***${COLOR_RESET}" 1>&2 - exit 1 -fi -if [[ ! -z "$TEST_REPO_ARCH" ]]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -MODELDIR=${MODELDIR:=`pwd`/models} -DATADIR=${DATADIR:="/data/inferenceserver/${REPO_VERSION}"} -TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} -SERVER=${TRITON_DIR}/bin/tritonserver -BACKEND_DIR=${TRITON_DIR}/backends - -# PyTorch on SBSA requires libgomp to be loaded first. See the following -# GitHub issue for more information: -# https://github.com/pytorch/pytorch/issues/2575 -arch=`uname -m` -echo -e "${COLOR_DARK}Detected architecture: ${arch}${COLOR_RESET}" -if [[ "${arch}" == "aarch64" ]]; then - SERVER_LD_PRELOAD=/usr/lib/$(uname -m)-linux-gnu/libgomp.so.1 - echo -e "${COLOR_DARK}SERVER_LD_PRELOAD=${SERVER_LD_PRELOAD}${COLOR_RESET}" -fi - -# If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="pytorch"} -export BACKENDS - -# Copy the models into the model repository -echo -e "${COLOR_DARK}Setting up model repository in ${MODELDIR}${COLOR_RESET}" -rm -rf ${MODELDIR} && mkdir -p ${MODELDIR} -models=( - "torch_aoti_complex_index" - "torch_aoti_complex_named" - "torch_aoti_int8_int8" - "torch_aoti_int16_int16" - "torch_aoti_int32_int32" - "torch_aoti_int64_int64" - "torch_aoti_float16_float16" - "torch_aoti_float32_float32" - "torchvision_aoti" -) -for model in "${models[@]}"; do - cp -r ${DATADIR}/qa_model_repository/${model} ${MODELDIR}/${model} - echo -e "${COLOR_DARK}ls ${MODELDIR}/${model}${COLOR_RESET}" - ls -lha ${MODELDIR}/${model} -done -echo -e "${COLOR_DARK}ls ${MODELDIR}${COLOR_RESET}" -ls -lha ${MODELDIR} - -SERVER_ARGS="--model-repository=${MODELDIR} --log-verbose=1" -SERVER_LOG="./torch_aoti_complex_named-server.log" -CLIENT_LOG="./torch_aoti_complex_named-client.log" - -echo -e "${COLOR_DARK}Running ${SERVER} with model repository ${MODELDIR}${COLOR_RESET}" -run_server -if [[ "${SERVER_PID}" -eq 0 ]]; then - echo -e "${COLOR_ERROR}\n***\n*** Failed to start ${SERVER}\n***${COLOR_RESET}" &1>2 - cat ${SERVER_LOG} &1>2 - echo -e "\n" &1>2 - exit 1 -fi - -# Install torch framework -echo -e "${COLOR_DARK}Installing PyTorch framework required by tests${COLOR_RESET}" -pip install torch - -# Run the Tests -TEST_NAME="torch_aoti_infer_test" -python3 ./${TEST_NAME}.py >> ${CLIENT_LOG} 2>&1 -EXIT_CODE=$? -if [[ ${EXIT_CODE} -ne 0 ]]; then - echo -e "${COLOR_ERROR}\n***\n*** Test '${TEST_NAME}' Failed with exit code ${EXIT_CODE}\n***${COLOR_RESET}" &1>2 - cat ${CLIENT_LOG} &1>2 - echo -e "\n" &1>2 - RET=1 -else - echo -e "${COLOR_INFO}\n***\n*** Test '${TEST_NAME}' Passed\n***${COLOR_RESET}" -fi - -# Cleanup -echo -e "${COLOR_DARK}Killing server (pid: ${SERVER_PID})${COLOR_RESET}" -kill -s SIGINT ${SERVER_PID} -wait ${SERVER_PID} || true -echo -e "${COLOR_DARK}Removing model repository${COLOR_RESET}" -for model in "${models[@]}"; do - rm -rf ${MODELDIR}/${model} -done - -# Report results and exit. -if [[ ${RET} -ne 0 ]]; then - echo -e "${COLOR_ERROR}\n***\n*** Test Suite FAILED\n***${COLOR_RESET}" &1>2 -else - echo -e "${COLOR_SUCCESS}\n***\n*** Test Suite PASSED\n***${COLOR_RESET}" -fi - -exit ${RET} diff --git a/qa/L0_torch_aoti/torch_aoti_infer_test.py b/qa/L0_torch_aoti/torch_aoti_infer_test.py deleted file mode 100755 index 2b93f31a48..0000000000 --- a/qa/L0_torch_aoti/torch_aoti_infer_test.py +++ /dev/null @@ -1,284 +0,0 @@ -#!/usr/bin/python -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import test_util as tu -import torch -import tritonclient.http as http - - -class TorchAotiTest(tu.TestResultCollector): - def _get_complex_input_shape(self): - return (1, 16) - - def _get_complex_output_shape(self): - return (1, 16) - - def _get_complex_input_data(self, shape): - return [ - torch.randint(low=0, high=127, size=shape, dtype=torch.int8).numpy(), - torch.randint(low=0, high=127, size=shape, dtype=torch.int8).numpy(), - torch.randint(low=0, high=127, size=shape, dtype=torch.int8).numpy(), - torch.randint(low=0, high=127, size=shape, dtype=torch.int8).numpy(), - ] - - def _get_simple_input_data(self, shape, io_type): - if io_type in [torch.int8, torch.int16, torch.int32, torch.int64]: - return torch.randint(low=0, high=127, size=shape, dtype=io_type).numpy() - elif io_type in [torch.float16, torch.float32, torch.float64]: - return torch.randn(size=shape, dtype=io_type).numpy() - else: - raise ValueError(f"Unsupported data type: {io_type}") - - def _get_torchvision_input_data(self, shape): - return torch.randn(size=shape, dtype=torch.float32).numpy() - - def _dtype_to_triton_dtype(self, dtype): - if dtype == torch.int8: - return "INT8" - elif dtype == torch.int16: - return "INT16" - elif dtype == torch.int32: - return "INT32" - elif dtype == torch.int64: - return "INT64" - elif dtype == torch.float16: - return "FP16" - elif dtype == torch.float32: - return "FP32" - else: - raise ValueError(f"Unsupported data type: {dtype}") - - def _get_simple_model_name(self, io_type): - if io_type == torch.int8: - return "torch_aoti_int8_int8" - elif io_type == torch.int16: - return "torch_aoti_int16_int16" - elif io_type == torch.int32: - return "torch_aoti_int32_int32" - elif io_type == torch.int64: - return "torch_aoti_int64_int64" - elif io_type == torch.float16: - return "torch_aoti_float16_float16" - elif io_type == torch.float32: - return "torch_aoti_float32_float32" - else: - raise ValueError(f"Unsupported data type: {io_type}") - - def test_complex_index(self): - MODEL_NAME = "torch_aoti_complex_index" - INPUT_SHAPE = self._get_complex_input_shape() - OUTPUT_SHAPE = self._get_complex_output_shape() - - input_data = self._get_complex_input_data(INPUT_SHAPE) - - with http.InferenceServerClient("localhost:8000") as client: - inputs = [ - http.InferInput("INPUT__0", input_data[0].shape, "INT8"), - http.InferInput("INPUT__1", input_data[1].shape, "INT8"), - http.InferInput("INPUT__2", input_data[2].shape, "INT8"), - http.InferInput("INPUT__3", input_data[3].shape, "INT8"), - ] - - inputs[0].set_data_from_numpy(input_data[0], binary_data=True) - inputs[1].set_data_from_numpy(input_data[1], binary_data=True) - inputs[2].set_data_from_numpy(input_data[2], binary_data=True) - inputs[3].set_data_from_numpy(input_data[3], binary_data=True) - - output_names = [ - "OUTPUT__0", - "OUTPUT__1", - "OUTPUT__2", - "OUTPUT__3", - "OUTPUT__4", - "OUTPUT__5", - ] - - outputs = [] - for output_name in output_names: - outputs.append(http.InferRequestedOutput(output_name, binary_data=True)) - - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) - - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) - - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - - self.assertTrue((output_data[0] == (input_data[0] + input_data[1])).all()) - self.assertTrue((output_data[1] == input_data[0] - input_data[1]).all()) - self.assertTrue((output_data[2] == input_data[0]).all()) - self.assertTrue((output_data[3] == input_data[1]).all()) - self.assertTrue((output_data[4] == input_data[2]).all()) - self.assertTrue((output_data[5] == input_data[3]).all()) - - def test_complex_named(self): - MODEL_NAME = "torch_aoti_complex_named" - INPUT_SHAPE = self._get_complex_input_shape() - OUTPUT_SHAPE = self._get_complex_output_shape() - - input_data = self._get_complex_input_data(INPUT_SHAPE) - - with http.InferenceServerClient("localhost:8000") as client: - inputs = [ - http.InferInput("ARGS[0]", input_data[0].shape, "INT8"), - http.InferInput("ARGS[1]", input_data[1].shape, "INT8"), - http.InferInput("ARGS[2][option1]", input_data[2].shape, "INT8"), - http.InferInput("ARGS[2][option2]", input_data[3].shape, "INT8"), - ] - - inputs[0].set_data_from_numpy(input_data[0], binary_data=True) - inputs[1].set_data_from_numpy(input_data[1], binary_data=True) - inputs[2].set_data_from_numpy(input_data[2], binary_data=True) - inputs[3].set_data_from_numpy(input_data[3], binary_data=True) - - output_names = [ - "RESULT[AAA]", - "RESULT[BBB][0]", - "RESULT[BBB][1]", - "RESULT[CCC][option1]", - "RESULT[CCC][option2]", - "RESULT[ZZZ]", - ] - - outputs = [] - for output_name in output_names: - outputs.append(http.InferRequestedOutput(output_name, binary_data=True)) - - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) - - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) - - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - - self.assertTrue((output_data[0] == (input_data[0] + input_data[1])).all()) - self.assertTrue((output_data[1] == input_data[0]).all()) - self.assertTrue((output_data[2] == input_data[1]).all()) - self.assertTrue((output_data[3] == input_data[2]).all()) - self.assertTrue((output_data[4] == input_data[3]).all()) - self.assertTrue((output_data[5] == (input_data[0] - input_data[1])).all()) - - def test_simple_model(self): - io_types = [ - torch.int8, - torch.int16, - torch.int32, - torch.int64, - torch.float16, - torch.float32, - ] - for io_type in io_types: - MODEL_NAME = self._get_simple_model_name(io_type) - INPUT_SHAPE = (16,) - OUTPUT_SHAPE = (16,) - TRITON_IO_TYPE = self._dtype_to_triton_dtype(io_type) - - input_data = ( - self._get_simple_input_data(INPUT_SHAPE, io_type), - self._get_simple_input_data(INPUT_SHAPE, io_type), - ) - - with http.InferenceServerClient("localhost:8000") as client: - inputs = [ - http.InferInput("ARGS[0]", input_data[0].shape, TRITON_IO_TYPE), - http.InferInput("ARGS[1]", input_data[1].shape, TRITON_IO_TYPE), - ] - - inputs[0].set_data_from_numpy(input_data[0], binary_data=True) - inputs[1].set_data_from_numpy(input_data[1], binary_data=True) - - output_names = [ - "RESULT", - ] - - outputs = [] - for output_name in output_names: - outputs.append( - http.InferRequestedOutput(output_name, binary_data=True) - ) - - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) - - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) - - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - self.assertTrue((data == input_data[0] + input_data[1]).all()) - - def test_torchvision(self): - MODEL_NAME = "torchvision_aoti" - INPUT_SHAPE = (1, 3, 224, 224) - OUTPUT_SHAPE = (1, 1000) - - input_data = self._get_torchvision_input_data(INPUT_SHAPE) - input_data[0][0] = 1.0 - - with http.InferenceServerClient("localhost:8000") as client: - inputs = [ - http.InferInput("ARGS[0]", input_data.shape, "FP32"), - ] - - inputs[0].set_data_from_numpy(input_data, binary_data=True) - - output_names = [ - "RESULT", - ] - - outputs = [] - for output_name in output_names: - outputs.append(http.InferRequestedOutput(output_name, binary_data=True)) - - output_data = [] - results = client.infer(MODEL_NAME, inputs, outputs=outputs) - - for output_name in output_names: - output_data.append(results.as_numpy(output_name)) - - self.assertEqual(len(outputs), len(output_data)) - for data in output_data: - self.assertEqual(data.shape, OUTPUT_SHAPE) - output_tensor = torch.from_numpy(data) - self.assertTrue(torch.isfinite(output_tensor).all().item()) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_trace/test.sh b/qa/L0_trace/test.sh index 7b29ea017a..ce53ea8528 100755 --- a/qa/L0_trace/test.sh +++ b/qa/L0_trace/test.sh @@ -109,11 +109,6 @@ mkdir -p $MODELSDIR/custom_identity_int32/1 && (cd $MODELSDIR/custom_identity_in RET=0 -# set up identity_fp32 model -mkdir -p $MODELSDIR/identity_fp32/1 && \ - cp ../python_models/identity_fp32/model.py $MODELSDIR/identity_fp32/1/. && \ - cp ../python_models/identity_fp32/config.pbtxt $MODELSDIR/identity_fp32/. - # Helpers ======================================= function assert_curl_success { message="${1}" @@ -191,26 +186,6 @@ function send_inference_requests { done } -function run_stress_client { - stress_client="${1}" - client_log="${2}" - echo "Running stress test for 120 seconds..." - bash -c ' - # Handle SIGTERM (signal 15) and exit gracefully - trap "echo \"cleaning up stress client...\"; exit 0" SIGTERM - - while true; do - python3 "$1" >> "$2" - sleep 0.1 - done' _ "$stress_client" "$client_log" & CLIENT_PID=$! - sleep 120 - - set -e - kill $CLIENT_PID - wait $CLIENT_PID - set +e -} - #======================================= # start with trace-level=OFF @@ -1087,9 +1062,6 @@ for p in {1..10}; do sleep 10 done -# Wait for all traces to be collected -sleep 5 - if ! [[ -s collected_traces.json && `grep -c "\"parentSpanId\":\"\"" ./collected_traces.json` == 1 && `grep -c "\"parentSpanId\":\"b7ad6b7169242424\"" ./collected_traces.json` == 10 ]] ; then echo -e "\n***\n*** collected_traces.json should contain 11 OTel trace, but it is not. \n***" exit 1 @@ -1272,78 +1244,4 @@ set -e kill $SERVER_PID wait $SERVER_PID set +e - -# Long running stress test -# Triton trace mode -SERVER_ARGS="--model-control-mode=explicit \ - --model-repository=$MODELSDIR \ - --load-model=identity_fp32 \ - --trace-config mode=triton \ - --trace-config triton,file=./trace \ - --trace-config rate=1 \ - --trace-config level=TIMESTAMPS" -SERVER_LOG="./inference_server_triton_trace_stress.log" -CLIENT_LOG="./client_triton_trace_stress.log" -STRESS_CLIENT="./trace_stress_grpc_client.py" - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -# Run stress test -run_stress_client $STRESS_CLIENT $CLIENT_LOG - -set -e -if ! kill -0 ${SERVER_PID} > /dev/null 2>&1; then - echo -e "\n***\n*** Server stopped unexpectedly during stress test\n***" - cat $SERVER_LOG - RET=1 -else - kill $SERVER_PID - wait $SERVER_PID -fi -set +e - -# Opentelemetry trace mode -SERVER_ARGS="--model-control-mode=explicit \ - --model-repository=$MODELSDIR \ - --load-model=identity_fp32 \ - --trace-config level=TIMESTAMPS \ - --trace-config rate=1 \ - --trace-config mode=opentelemetry \ - --trace-config opentelemetry,resource=test.key=test.value \ - --trace-config opentelemetry,resource=service.name=test_triton \ - --trace-config opentelemetry,url=localhost:$OTLP_PORT/v1/traces" -SERVER_LOG="./inference_server_otel_trace_stress.log" -CLIENT_LOG="./client_otel_trace_stress.log" -STRESS_CLIENT="./trace_stress_grpc_client.py" - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -rm collected_traces.json -$OTEL_COLLECTOR --config ./trace-config.yaml >> $OTEL_COLLECTOR_LOG 2>&1 & COLLECTOR_PID=$! -# Run stress test -run_stress_client $STRESS_CLIENT $CLIENT_LOG - -set -e -kill $COLLECTOR_PID -wait $COLLECTOR_PID -if ! kill -0 ${SERVER_PID} > /dev/null 2>&1; then - echo -e "\n***\n*** Server stopped unexpectedly during stress test\n***" - cat $SERVER_LOG - RET=1 -else - kill $SERVER_PID - wait $SERVER_PID -fi -set +e - exit $RET diff --git a/qa/L0_trace/trace_stress_grpc_client.py b/qa/L0_trace/trace_stress_grpc_client.py deleted file mode 100755 index 99a676f150..0000000000 --- a/qa/L0_trace/trace_stress_grpc_client.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import random -import sys -import time -from functools import partial - -import numpy as np -import tritonclient.grpc as grpcclient - -if __name__ == "__main__": - # 1 ms cancellation timeout - client_timeout = 1 - url = "localhost:8001" - - try: - triton_client = grpcclient.InferenceServerClient(url=url) - except Exception as e: - print("context creation failed: " + str(e)) - sys.exit() - - model_name = "identity_fp32" - - # Infer - inputs = [] - - input_data = np.array( - [random.random() for i in range(50)], dtype=np.float32 - ).reshape(1, -1) - model_input = grpcclient.InferInput( - name="INPUT0", datatype="FP32", shape=input_data.shape - ) - model_input.set_data_from_numpy(input_data) - inputs.append(model_input) - - # Define the callback function. Note the last two parameters should be - # result and error. InferenceServerClient would povide the results of an - # inference as grpcclient.InferResult in result. For successful - # inference, error will be None, otherwise it will be an object of - # tritonclientutils.InferenceServerException holding the error details - def callback(user_data, result, error): - if error: - user_data.append(error) - else: - user_data.append(result) - - # list to hold the results of inference. - user_data = [] - - # Inference call - for _ in range(1000): - triton_client.async_infer( - model_name=model_name, - inputs=inputs, - callback=partial(callback, user_data), - client_timeout=client_timeout, - ) - - # Wait until the results are available in user_data - time_out = 20 - while (len(user_data) == 0) and time_out > 0: - time_out = time_out - 1 - time.sleep(1) - - print("results: ", len(user_data)) diff --git a/qa/L0_trt_bf16_dtype/trt_bf16_dtype_test.py b/qa/L0_trt_bf16_dtype/trt_bf16_dtype_test.py index c80c84d353..265c1930b0 100755 --- a/qa/L0_trt_bf16_dtype/trt_bf16_dtype_test.py +++ b/qa/L0_trt_bf16_dtype/trt_bf16_dtype_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -32,7 +32,6 @@ import unittest -import ml_dtypes import numpy as np import test_util as tu import tritonclient.http as client @@ -50,9 +49,8 @@ def _infer_helper(self, model_name, shape): inputs.append(client.InferInput("INPUT0", shape, "BF16")) inputs.append(client.InferInput("INPUT1", shape, "BF16")) - # BF16 inputs use ml_dtypes.bfloat16; numpy has no native BF16 dtype. - input0_data = np.ones(shape, dtype=ml_dtypes.bfloat16) - input1_data = np.ones(shape, dtype=ml_dtypes.bfloat16) + input0_data = np.ones(shape=shape).astype(np.float32) + input1_data = np.ones(shape=shape).astype(np.float32) inputs[0].set_data_from_numpy(input0_data, binary_data=True) inputs[1].set_data_from_numpy(input1_data, binary_data=True) @@ -65,8 +63,6 @@ def _infer_helper(self, model_name, shape): output0_data = results.as_numpy("OUTPUT0") output1_data = results.as_numpy("OUTPUT1") - self.assertEqual(output0_data.dtype, ml_dtypes.bfloat16) - self.assertEqual(output1_data.dtype, ml_dtypes.bfloat16) np.testing.assert_equal( output0_data, input0_data + input1_data, diff --git a/qa/L0_trt_dynamic_shape/test.sh b/qa/L0_trt_dynamic_shape/test.sh index 42928969fd..43a39dd199 100755 --- a/qa/L0_trt_dynamic_shape/test.sh +++ b/qa/L0_trt_dynamic_shape/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,10 +41,8 @@ fi TEST_RESULT_FILE='test_results.txt' export CUDA_VISIBLE_DEVICES=0 -pip3 install perf_analyzer - CLIENT_LOG="./client.log" -PERF_CLIENT=perf_analyzer +PERF_CLIENT=../clients/perf_client TRT_OP_TEST=trt_dynamic_shape_test.py DATADIR="./models" @@ -72,9 +70,11 @@ fi # Shape beyond the limits of optimization profile set +e $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32-4-32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 1 > ${CLIENT_LOG}_max 2>&1 -EXIT_CODE=$? -echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_max" -"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_max" 2>&1 || true +if [ $? -eq 0 ]; then + cat ${CLIENT_LOG}_max + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi EXPECTED_MESSAGE="model expected the shape of dimension 1 to be between 4 and 32 but received" if [ $(cat ${CLIENT_LOG}_max | grep "${EXPECTED_MESSAGE} 33" | wc -l) -eq 0 ]; then @@ -84,10 +84,11 @@ if [ $(cat ${CLIENT_LOG}_max | grep "${EXPECTED_MESSAGE} 33" | wc -l) -eq 0 ]; t fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32-4-32 --shape INPUT0:3 --shape INPUT1:3 -t 1 -p2000 -b 1 > ${CLIENT_LOG}_min 2>&1 -EXIT_CODE=$? -echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_min" -"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_min" 2>&1 || true - +if [ $? -eq 0 ]; then + cat ${CLIENT_LOG}_min + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi if [ $(cat ${CLIENT_LOG}_min | grep "${EXPECTED_MESSAGE} 3" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_min echo -e "\n***\n*** Test Failed\n***" @@ -330,10 +331,11 @@ if [ $? -ne 0 ]; then fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 6 > ${CLIENT_LOG}_static_fail 2>&1 -EXIT_CODE=$? -echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_static_fail" -"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_static_fail" 2>&1 || true - +if [ $? -eq 0 ]; then + ${CLIENT_LOG}_static_fail + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi if [ $(cat ${CLIENT_LOG}_static_fail | grep "inference request batch-size must be <= 5" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_static_fail echo -e "\n***\n*** Test Failed\n***" @@ -341,10 +343,11 @@ if [ $(cat ${CLIENT_LOG}_static_fail | grep "inference request batch-size must b fi $PERF_CLIENT -v -i grpc -u localhost:8001 -m plan_float32_float32_float32 --shape INPUT0:33 --shape INPUT1:33 -t 1 -p2000 -b 2 > ${CLIENT_LOG}_static_bs_2 2>&1 -EXIT_CODE=$? -echo "perf_analyzer exit code: ${EXIT_CODE}" >> "${CLIENT_LOG}_static_bs_2" -"${PERF_CLIENT}" --version >> "${CLIENT_LOG}_static_bs_2" 2>&1 || true - +if [ $? -eq 0 ]; then + ${CLIENT_LOG}_static_bs_2 + echo -e "\n***\n*** Test Failed\n***" + RET=1 +fi if [ $(cat ${CLIENT_LOG}_static_bs_2 | grep "model expected the shape of dimension 0 to be between 1 and 1 but received 2" | wc -l) -eq 0 ]; then cat ${CLIENT_LOG}_static_bs_2 echo -e "\n***\n*** Test Failed\n***" diff --git a/qa/L0_trt_reformat_free/test.sh b/qa/L0_trt_reformat_free/test.sh index 4cf3198ec2..2daf2f0648 100755 --- a/qa/L0_trt_reformat_free/test.sh +++ b/qa/L0_trt_reformat_free/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,7 +50,7 @@ rm -rf ${DATADIR} cp -r /data/inferenceserver/${REPO_VERSION}/qa_trt_format_model_repository/ ${DATADIR} SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=$DATADIR --allow-client-shm=true" +SERVER_ARGS="--model-repository=$DATADIR" source ../common/util.sh rm -f *.log* diff --git a/qa/L0_trt_shape_tensors/test.sh b/qa/L0_trt_shape_tensors/test.sh index 9d03791007..548ebb55af 100755 --- a/qa/L0_trt_shape_tensors/test.sh +++ b/qa/L0_trt_shape_tensors/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,18 +45,10 @@ CLIENT_LOG="./client.log" SHAPE_TENSOR_TEST=trt_shape_tensor_test.py SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --allow-client-shm=true" +SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" SERVER_LOG="./inference_server.log" source ../common/util.sh -if [ -z "$TEST_SYSTEM_SHARED_MEMORY" ]; then - TEST_SYSTEM_SHARED_MEMORY="0" -fi - -if [ -z "$TEST_CUDA_SHARED_MEMORY" ]; then - TEST_CUDA_SHARED_MEMORY="0" -fi - rm -fr *.log rm -fr models && mkdir models cp -r /data/inferenceserver/${REPO_VERSION}/qa_shapetensor_model_repository/* models/. @@ -228,9 +220,6 @@ for i in \ test_dynaseq_different_shape_values_parallel \ ;do SERVER_ARGS="--model-repository=`pwd`/models" - if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then - SERVER_ARGS="${SERVER_ARGS} --allow-client-shm=true" - fi SERVER_LOG="./$i.server.log" run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/L0_vertex_ai/test.sh b/qa/L0_vertex_ai/test.sh index c8da5ed57d..7403bf14cf 100755 --- a/qa/L0_vertex_ai/test.sh +++ b/qa/L0_vertex_ai/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -44,7 +44,7 @@ export CUDA_VISIBLE_DEVICES=0 RET=0 -rm -rf multi_models single_model restricted_single_model +rm -rf multi_models single_model rm -f *.log rm -f *.out @@ -68,11 +68,6 @@ mkdir multi_models && \ mkdir single_model && \ cp -r multi_models/addsub single_model/. -# Set up single-model Python repository used by restricted API regression -mkdir -p restricted_single_model/identity_fp32/1 && \ - cp ../python_models/identity_fp32/config.pbtxt restricted_single_model/identity_fp32/ && \ - cp ../python_models/identity_fp32/model.py restricted_single_model/identity_fp32/1/ - # Use Vertex AI's health endpoint to check server status # Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on # success, 1 on failure @@ -139,7 +134,7 @@ if [ "$SERVER_PID" == "0" ]; then exit 1 fi kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID set +e # Expect no message regarding Vertex AI as it is disabled grep "failed to start Vertex AI service" $SERVER_LOG @@ -162,7 +157,7 @@ if [ "$SERVER_PID" == "0" ]; then exit 1 fi kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID set +e grep "Started Vertex AI HTTPService at" $SERVER_LOG if [ $? -ne 0 ]; then @@ -191,7 +186,7 @@ if [ "$SERVER_PID" == "0" ]; then exit 1 fi kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID set +e grep "Started Vertex AI HTTPService at" $SERVER_LOG if [ $? -ne 0 ]; then @@ -225,7 +220,7 @@ if [ "$SERVER_PID" == "0" ]; then exit 1 fi kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID set +e # Expect no message regarding Vertex AI as it is disabled grep "failed to start Vertex AI service" $SERVER_LOG @@ -336,7 +331,7 @@ fi set -e kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID # # AIP_STORAGE_URI / AIP_HTTP_PORT @@ -375,7 +370,7 @@ fi set -e kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID # # default model @@ -458,7 +453,7 @@ else fi set -e kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID # Test default model as well as multi model SERVER_ARGS="--vertex-ai-default-model=addsub" @@ -699,247 +694,25 @@ else fi set -e -# repository control via redirect is blocked unconditionally +# repository control (expect error) rm -f ./curl.out set +e code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/repository/models/subadd/unload" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model unload via redirect to return 403 (got $code)\n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# -# Restricted API regression for Vertex redirect -# -unset_vertex_variables -export AIP_PREDICT_ROUTE="/predict" -export AIP_HEALTH_ROUTE="/health" - -SERVER_LOG="vertex_restricted_api_testing_server.log" -SERVER_ARGS="--log-verbose=1 --allow-vertex-ai=true \ - --model-repository=restricted_single_model \ - --vertex-ai-default-model=identity_fp32 \ - --http-restricted-api=metadata,model-config,model-repository,statistics,shared-memory:X-Vertex-Restricted=secret" - -run_server_nowait -vertex_ai_wait_for_server_ready $SERVER_PID 10 -if [ "$WAIT_RET" != "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - kill $SERVER_PID - wait $SERVER_PID - cat $SERVER_LOG - exit 1 -fi - -# Baseline infer remains available without restricted header -set +e -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "Content-Type: application/json" -d'{"inputs":[{"name":"INPUT0","datatype":"FP32","shape":[1,1],"data":[42.0]}],"outputs":[{"name":"OUTPUT0"}]}' localhost:8080/predict` -if [ "$code" != "200" ]; then +if [ "$code" == "200" ]; then cat ./curl.out - echo -e "\n***\n*** Failed. Expected /predict inference succeeds without restricted header\n***" + echo -e "\n***\n*** Test Failed\n***" RET=1 else - grep "OUTPUT0" ./curl.out + grep "explicit model load / unload is not allowed" ./curl.out if [ $? -ne 0 ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected inference output is returned\n***" + echo -e "\n***\n*** Failed. Expected error on model control\n***" RET=1 fi fi set -e -# Redirected read-only APIs should be blocked without restricted header -set +e -for redirect_endpoint in \ - "v2" \ - "v2/models/identity_fp32/config" \ - "v2/repository/index" \ - "v2/systemsharedmemory/status" \ - "v2/cudasharedmemory/status" -do - rm -f ./curl.out - code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: ${redirect_endpoint}" localhost:8080/predict` - if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected ${redirect_endpoint} is restricted\n***" - RET=1 - fi -done - -# Mutating shared memory operations are blocked through redirect -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/systemsharedmemory/region/test/register" -H "Content-Type: application/json" -d '{"key":"test_shm","byte_size":1024}' localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected shared memory register is blocked via redirect\n***" - RET=1 -fi - -# Model load redirect is unconditionally blocked through the prediction endpoint -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/repository/models/identity_fp32/load" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model load via redirect is blocked\n***" - RET=1 -fi - -# Model unload redirect is unconditionally blocked through the prediction endpoint -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/repository/models/identity_fp32/unload" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model unload via redirect is blocked\n***" - RET=1 -fi - -# Statistics redirect without restricted header should be blocked -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/models/identity_fp32/stats" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model statistics via redirect is restricted\n***" - RET=1 -fi -set -e - -# Restricted header should allow read-only handler invocation -set +e -for redirect_endpoint in \ - "v2" \ - "v2/models/identity_fp32/config" \ - "v2/repository/index" \ - "v2/systemsharedmemory/status" \ - "v2/cudasharedmemory/status" -do - rm -f ./curl.out - code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: ${redirect_endpoint}" \ - -H "X-Vertex-Restricted: secret" localhost:8080/predict` - if [ "$code" != "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected ${redirect_endpoint} passes with restricted header\n***" - RET=1 - fi -done - -# Mutating shared memory operations remain blocked even with valid header -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/systemsharedmemory/region/test/register" -H "X-Vertex-Restricted: secret" -H "Content-Type: application/json" -d '{"key":"test_shm","byte_size":1024}' localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected shared memory register is blocked via redirect even with valid header\n***" - RET=1 -fi - -# Model load remains blocked even with valid restricted header -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/repository/models/identity_fp32/load" -H "X-Vertex-Restricted: secret" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model load is blocked via redirect even with valid header\n***" - RET=1 -fi - -# Model unload remains blocked even with valid restricted header -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/repository/models/identity_fp32/unload" -H "X-Vertex-Restricted: secret" localhost:8080/predict` -if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model unload is blocked via redirect even with valid header\n***" - RET=1 -fi - -# Statistics redirect with restricted header should pass -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: v2/models/identity_fp32/stats" -H "X-Vertex-Restricted: secret" localhost:8080/predict` -if [ "$code" == "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected model statistics passes with restricted header\n***" - RET=1 -fi - -# Wrong restricted header should reject the request -set +e -for redirect_endpoint in \ - "v2" \ - "v2/models/identity_fp32/config" \ - "v2/repository/index" \ - "v2/systemsharedmemory/status" \ - "v2/cudasharedmemory/status" \ - "v2/systemsharedmemory/region/test/register" -do - rm -f ./curl.out - if [ "$redirect_endpoint" != "v2/systemsharedmemory/region/test/register" ]; then - code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: ${redirect_endpoint}" \ - -H "X-Vertex-Restricted: invalid" localhost:8080/predict` - else - code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "X-Vertex-Ai-Triton-Redirect: ${redirect_endpoint}" \ - -H "X-Vertex-Restricted: invalid" -H "Content-Type: application/json" -d '{"key":"test_shm","byte_size":1024}' localhost:8080/predict` - fi - if [ "$code" != "403" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected invalid restricted header value is rejected for ${redirect_endpoint}\n***" - RET=1 - fi -done -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# -# HTTP max input size enforcement on Vertex AI endpoint -# -unset_vertex_variables -export AIP_PREDICT_ROUTE="/predict" -export AIP_HEALTH_ROUTE="/health" - -SERVER_LOG="vertex_max_input_size_server.log" -SERVER_ARGS="--allow-vertex-ai=true \ - --model-repository=restricted_single_model \ - --vertex-ai-default-model=identity_fp32 \ - --http-max-input-size=128" -run_server_nowait -vertex_ai_wait_for_server_ready $SERVER_PID 10 -if [ "$WAIT_RET" != "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - kill $SERVER_PID - wait $SERVER_PID - cat $SERVER_LOG - exit 1 -fi - -set +e - -# Small payload under 128 bytes should succeed -rm -f ./curl.out -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "Content-Type: application/json" -d'{"inputs":[{"name":"INPUT0","datatype":"FP32","shape":[1,1],"data":[1.0]}],"outputs":[{"name":"OUTPUT0"}]}' localhost:8080/predict` -if [ "$code" != "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected small payload to succeed on Vertex AI endpoint (got $code)\n***" - RET=1 -fi - -# Large payload over 128 bytes should be rejected -rm -f ./curl.out -LARGE_PAYLOAD='{"inputs":[{"name":"INPUT0","datatype":"FP32","shape":[1,16],"data":[1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0,11.0,12.0,13.0,14.0,15.0,16.0]}],"outputs":[{"name":"OUTPUT0"}]}' -code=`curl -s -w %{http_code} -o ./curl.out -X POST -H "Content-Type: application/json" -d"$LARGE_PAYLOAD" localhost:8080/predict` -if [ "$code" == "200" ]; then - cat ./curl.out - echo -e "\n***\n*** Failed. Expected oversized payload to be rejected on Vertex AI endpoint\n***" - RET=1 -fi - -set -e - kill $SERVER_PID -wait $SERVER_PID +wait $SERVE_PID if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" diff --git a/qa/L0_warmup/test.sh b/qa/L0_warmup/test.sh index 84a0752229..c3e6885062 100755 --- a/qa/L0_warmup/test.sh +++ b/qa/L0_warmup/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -420,7 +420,7 @@ wait $SERVER_PID # Test the onnx model to verify that the memory type of the output tensor # remains unchanged with the warmup setting pip3 uninstall -y torch -pip3 install torch -f https://download.pytorch.org/whl/cu130 +pip3 install torch==2.3.1+cu118 -f https://download.pytorch.org/whl/torch_stable.html rm -fr models && mkdir models cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. diff --git a/qa/common/check_copyright.py b/qa/common/check_copyright.py index de3bdc79b4..99606a1c81 100755 --- a/qa/common/check_copyright.py +++ b/qa/common/check_copyright.py @@ -75,7 +75,6 @@ "qa/L0_model_config/cli_messages/cli_override/expected", "qa/L0_model_config/cli_messages/cli_deprecation/expected", "qa/L0_model_config/model_metrics", - "qa/L0_model_config/custom_parameters", "qa/L0_model_namespacing/test_duplication", "qa/L0_model_namespacing/test_dynamic_resolution", "qa/L0_model_namespacing/test_ensemble_duplication", diff --git a/qa/common/check_valgrind_log.py b/qa/common/check_valgrind_log.py index 8a59dd3e9c..84eecd4989 100755 --- a/qa/common/check_valgrind_log.py +++ b/qa/common/check_valgrind_log.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -58,7 +58,7 @@ def check_valgrind_log(log_file): a list of the leak records as strings """ - with open(log_file, "r") as f: + with open(args.input_log_file, "r") as f: logs = f.read() # Find the pid and start and end of definite leak reports diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index db0869ef38..d53702d604 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -1,4 +1,4 @@ -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,14 +28,13 @@ from typing import List # Common utilities for model generation scripts -import ml_dtypes import numpy as np np_dtype_string = np.dtype(object) # Numpy does not support the BF16 datatype natively. -# We use ml_dtypes.bfloat16 for BF16. -np_dtype_bfloat16 = ml_dtypes.bfloat16 +# We use this dummy dtype as a representative for BF16. +np_dtype_bfloat16 = np.dtype([("bf16", object)]) def np_to_onnx_dtype(np_dtype): @@ -63,8 +62,6 @@ def np_to_onnx_dtype(np_dtype): return onnx.TensorProto.DOUBLE elif np_dtype == np_dtype_string: return onnx.TensorProto.STRING - elif np_dtype == np_dtype_bfloat16: - return onnx.TensorProto.BFLOAT16 return None diff --git a/qa/common/gen_ensemble_model_utils.py b/qa/common/gen_ensemble_model_utils.py index 1dae50a380..fec8f0bf92 100755 --- a/qa/common/gen_ensemble_model_utils.py +++ b/qa/common/gen_ensemble_model_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -761,7 +761,7 @@ def create_ensemble_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir @@ -834,7 +834,7 @@ def create_ensemble_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -867,7 +867,7 @@ def create_identity_ensemble_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir @@ -915,7 +915,7 @@ def create_identity_ensemble_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -937,7 +937,7 @@ def create_sequence_ensemble_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir @@ -979,7 +979,7 @@ def create_sequence_ensemble_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -1015,7 +1015,7 @@ def create_nop_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -1059,7 +1059,7 @@ def create_nop_tunnel_modelconfig(models_dir, tensor_shape, tensor_dtype): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -1084,7 +1084,7 @@ def create_nop_tunnel_modelconfig(models_dir, tensor_shape, tensor_dtype): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 4d491fa2a1..1dda5b72cf 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,15 +34,15 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=26.05} +TRITON_VERSION=${TRITON_VERSION:=25.03} # Set the CUDA device to use -NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:=0} +CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image TENSORRT_IMAGE=${TENSORRT_IMAGE:=nvcr.io/nvidia/tensorrt:$TRITON_VERSION-py3-igpu} UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} # Set CI specific parameters -DOCKER_GPU_ARGS=${DOCKER_GPU_ARGS:-$([[ -v RUNNER_GPUS && $RUNNER_GPUS =~ ^[0-9] ]] && eval $NV_DOCKER_ARGS || echo "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES" )} +DOCKER_GPU_ARGS=${DOCKER_GPU_ARGS:-$([[ $RUNNER_GPUS =~ ^[0-9] ]] && eval $NV_DOCKER_ARGS || echo "--gpus device=$CUDA_DEVICE" )} ############################################################################ # Check if Docker volume exists @@ -142,7 +142,6 @@ docker pull $TENSORRT_IMAGE docker run $DOCKER_GPU_ARGS \ --rm -v $DOCKER_VOLUME:/mnt \ - -e TRT_VERBOSE \ $TENSORRT_IMAGE bash -xe $VOLUME_SRCDIR/$TRT_MODEL_SCRIPT # Copy generated models to /tmp/ if not running in CI diff --git a/qa/common/gen_qa_custom_ops b/qa/common/gen_qa_custom_ops new file mode 100755 index 0000000000..304e126c34 --- /dev/null +++ b/qa/common/gen_qa_custom_ops @@ -0,0 +1,149 @@ +#!/bin/bash +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +############################################################################ +## This script generates custom operations needed by some of the +## Triton's CI tests. Generating these custom operations requires +## using the Pytorch container. +## +## 1. Update PYTORCH_IMAGE to match what is being +## used by the Triton release being tested. +## +## 2. Run this script to create /tmp/qa_custom_ops. +## +############################################################################ + +TRITON_VERSION=${TRITON_VERSION:=25.03} +NVIDIA_UPSTREAM_VERSION=${NVIDIA_UPSTREAM_VERSION:=$TRITON_VERSION} +PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$NVIDIA_UPSTREAM_VERSION-py3} +UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} + +CUDA_DEVICE=${NV_GPU:=0} + +DOCKER_GPU_ARGS=${DOCKER_GPU_ARGS:-$([[ $RUNNER_GPUS =~ ^[0-9] ]] && eval $NV_DOCKER_ARGS || echo "--gpus device=$CUDA_DEVICE" )} + +############################################################################ +# Check if Docker volume exists +############################################################################ +CI_JOB_ID=${CI_JOB_ID:=$(date +%Y%m%d_%H%M)} +DOCKER_VOLUME=${DOCKER_VOLUME:=volume.gen_qa_custom_ops.${CI_JOB_ID}} +RUNNER_ID=${RUNNER_ID:=0} +PROJECT_NAME=${PROJECT_NAME:=tritonserver} +DOCKER_VOLUME_CONTAINER=${DOCKER_VOLUME}.gen_qa_custom_ops.${CI_JOB_ID} + +if ! docker volume inspect $DOCKER_VOLUME > /dev/null 2>&1; then + echo -e "\033[34m[ INFO ] - Docker volume $DOCKER_VOLUME does not exist. Creating... \033[0m " + docker volume create $DOCKER_VOLUME --label RUNNER_ID=$RUNNER_ID --label PROJECT_NAME=$PROJECT_NAME + docker volume inspect $DOCKER_VOLUME +else + echo -e "\033[34m[ INFO ] - Docker volume in use: $DOCKER_VOLUME \033[0m " + docker volume inspect $DOCKER_VOLUME +fi + + +docker run \ + --rm \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + -v $DOCKER_VOLUME:/mnt \ + -w /mnt/$CI_JOB_ID \ + $UBUNTU_IMAGE \ + mkdir -p gen_srcdir ${TRITON_VERSION} + +docker create \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + --name $DOCKER_VOLUME_CONTAINER \ + -v $DOCKER_VOLUME:/mnt \ + -w /mnt/$CI_JOB_ID \ + $UBUNTU_IMAGE + +docker cp . $DOCKER_VOLUME_CONTAINER:/mnt/$CI_JOB_ID/gen_srcdir + +### +VOLUME_BUILD_DIR=${VOLUME_BUILD_DIR:=/mnt/$CI_JOB_ID} +VOLUME_SRCDIR=${VOLUME_SRCDIR:=$VOLUME_BUILD_DIR/gen_srcdir} +VOLUME_DESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_custom_ops + + +docker run --rm -v $DOCKER_VOLUME:/mnt -w /mnt/$CI_JOB_ID $UBUNTU_IMAGE \ +mkdir -p \ +$VOLUME_DESTDIR/libtorch_custom_ops + +PYTSCRIPT=gen.PyTorch.gen_qa_custom_ops.cmds + +# PyTorch + +cat > $PYTSCRIPT < $OPENVINOSCRIPT < /dev/null 2>&1; then + echo -e "\033[34m[ INFO ] - Docker volume $DOCKER_VOLUME does not exist. Creating... \033[0m " + docker volume create $DOCKER_VOLUME --label RUNNER_ID=$RUNNER_ID --label PROJECT_NAME=$PROJECT_NAME + docker volume inspect $DOCKER_VOLUME +else + echo -e "\033[34m[ INFO ] - Docker volume in use: $DOCKER_VOLUME \033[0m " + docker volume inspect $DOCKER_VOLUME +fi + + +docker run \ + --rm \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + -v $DOCKER_VOLUME:/mnt \ + -w /mnt/$CI_JOB_ID \ + $UBUNTU_IMAGE \ + mkdir -p gen_srcdir ${TRITON_VERSION} + +docker create \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + --name $DOCKER_VOLUME_CONTAINER \ + -v $DOCKER_VOLUME:/mnt \ + -w /mnt/$CI_JOB_ID \ + $UBUNTU_IMAGE + +docker cp . $DOCKER_VOLUME_CONTAINER:/mnt/$CI_JOB_ID/gen_srcdir + +VOLUME_BUILD_DIR=${VOLUME_BUILD_DIR:=/mnt/$CI_JOB_ID} +VOLUME_SRCDIR=${VOLUME_SRCDIR:=$VOLUME_BUILD_DIR/gen_srcdir} +VOLUME_DESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_model_repository +VOLUME_VARDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_variable_model_repository +VOLUME_IDENTITYDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_identity_model_repository +VOLUME_IDENTITYBIGDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_identity_big_model_repository +VOLUME_SHAPEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_shapetensor_model_repository +VOLUME_RESHAPEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_reshape_model_repository +VOLUME_SEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_sequence_model_repository +VOLUME_DYNASEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_dyna_sequence_model_repository +VOLUME_DYNASEQIMPLICITDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_dyna_sequence_implicit_model_repository +VOLUME_VARSEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_variable_sequence_model_repository +VOLUME_ENSEMBLEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_ensemble_model_repository +VOLUME_NOSHAPEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_noshape_model_repository +VOLUME_PLGDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_trt_plugin_model_repository +VOLUME_RAGGEDDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_ragged_model_repository +VOLUME_FORMATDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_trt_format_model_repository +VOLUME_DATADEPENDENTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_trt_data_dependent_model_repository +VOLUME_IMPLICITSEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_sequence_implicit_model_repository +VOLUME_VARIMPLICITSEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_variable_sequence_implicit_model_repository +VOLUME_INITIALSTATEIMPLICITSEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_sequence_initial_state_implicit_model_repository +VOLUME_VARINITIALSTATEIMPLICITSEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_variable_sequence_initial_state_implicit_model_repository +VOLUME_TORCHTRTDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/torchtrt_model_store +VOLUME_SCALARMODELSDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_scalar_models +VOLUME_IMAGEMODELSDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_dynamic_batch_image_model_repository + + +docker run \ + --rm \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + -v $DOCKER_VOLUME:/mnt \ + -w /mnt/$CI_JOB_ID \ + $UBUNTU_IMAGE \ + mkdir -p \ + $VOLUME_BUILD_DIR \ + $VOLUME_SRCDIR \ + $VOLUME_DESTDIR \ + $VOLUME_VARDESTDIR \ + $VOLUME_IDENTITYDESTDIR \ + $VOLUME_SIGDEFDESTDIR \ + $VOLUME_IDENTITYBIGDESTDIR \ + $VOLUME_TFPARAMETERSDESTDIR \ + $VOLUME_SHAPEDESTDIR \ + $VOLUME_RESHAPEDESTDIR \ + $VOLUME_SEQDESTDIR \ + $VOLUME_DYNASEQDESTDIR \ + $VOLUME_DYNASEQIMPLICITDESTDIR \ + $VOLUME_VARSEQDESTDIR \ + $VOLUME_ENSEMBLEDESTDIR \ + $VOLUME_NOSHAPEDESTDIR \ + $VOLUME_PLGDESTDIR \ + $VOLUME_RAGGEDDESTDIR \ + $VOLUME_FORMATDESTDIR \ + $VOLUME_DATADEPENDENTDIR \ + $VOLUME_IMPLICITSEQDESTDIR \ + $VOLUME_VARIMPLICITSEQDESTDIR \ + $VOLUME_INITIALSTATEIMPLICITSEQDESTDIR \ + $VOLUME_VARINITIALSTATEIMPLICITSEQDESTDIR \ + $VOLUME_TORCHTRTDESTDIR \ + $VOLUME_SCALARMODELSDESTDIR \ + $VOLUME_IMAGEMODELSDESTDIR + +ONNXSCRIPT=gen.ONNXRuntime.gen_qa_model_repository.cmds +OPENVINOSCRIPT=gen.OpenVINO.gen_qa_model_repository.cmds +TORCHSCRIPT=gen.PyTorch.gen_qa_model_repository.cmds +TRTSCRIPT=gen.TensorRT.gen_qa_model_repository.cmds + +# OPENVINO +# +# OpenVINO is not available on ARM so skip +if [[ "aarch64" != $(uname -m) ]] ; then + +cat > $OPENVINOSCRIPT < $ONNXSCRIPT < $ONNXSCRIPT < $TORCHSCRIPT < $TORCHSCRIPT < $TRTSCRIPT < $TRTSCRIPT < -# branch matching the runtime TRT version. If the branch is not published -# yet for this TRT version, the plugin model generation is skipped with a -# warning -- L0_trt_plugin will report missing artifacts but the rest of -# the QA model repository is still produced. -TRT_BRANCH=\$(echo \${TRT_VERSION} | cut -d . -f -2) -TRTSRC=/workspace/TensorRT -rm -rf \${TRTSRC} -if git clone --depth 1 -b release/\${TRT_BRANCH} \ - https://github.com/NVIDIA/TensorRT.git \${TRTSRC}; then - cd \${TRTSRC}/samples/python/onnx_custom_plugin && \ - rm -rf build && mkdir build && cd build && cmake .. && make -j && \ - cp libcustomHardmaxPlugin.so ${TRITON_MDLS_QA_TRT_PLUGIN_MODEL}/. - LD_PRELOAD=${TRITON_MDLS_QA_TRT_PLUGIN_MODEL}/libcustomHardmaxPlugin.so \ - python3 ${TRITON_MDLS_SRC_DIR}/gen_qa_trt_plugin_models.py \ - --models_dir=${TRITON_MDLS_QA_TRT_PLUGIN_MODEL} - chmod -R 777 ${TRITON_MDLS_QA_TRT_PLUGIN_MODEL} +python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --tensorrt-shape-io --models_dir=$VOLUME_SHAPEDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --tensorrt-shape-io --models_dir=$VOLUME_SHAPEDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_dyna_sequence_models.py --tensorrt-shape-io --models_dir=$VOLUME_SHAPEDESTDIR +chmod -R 777 $VOLUME_SHAPEDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_models.py --tensorrt --models_dir=$VOLUME_DESTDIR +chmod -R 777 $VOLUME_DESTDIR +python3 $VOLUME_SRCDIR/gen_qa_models.py --tensorrt --variable --models_dir=$VOLUME_VARDESTDIR +chmod -R 777 $VOLUME_VARDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --tensorrt --models_dir=$VOLUME_IDENTITYDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --tensorrt-compat --models_dir=$VOLUME_IDENTITYDESTDIR +chmod -R 777 $VOLUME_IDENTITYDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --tensorrt-big --models_dir=$VOLUME_IDENTITYBIGDESTDIR +chmod -R 777 $VOLUME_IDENTITYBIGDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_reshape_models.py --tensorrt --variable --models_dir=$VOLUME_RESHAPEDESTDIR +chmod -R 777 $VOLUME_RESHAPEDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --tensorrt --models_dir=$VOLUME_SEQDESTDIR +chmod -R 777 $VOLUME_SEQDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_implicit_models.py --tensorrt --models_dir=$VOLUME_IMPLICITSEQDESTDIR +chmod -R 777 $VOLUME_IMPLICITSEQDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_implicit_models.py --tensorrt --variable --models_dir=$VOLUME_VARIMPLICITSEQDESTDIR +chmod -R 777 $VOLUME_VARIMPLICITSEQDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_dyna_sequence_models.py --tensorrt --models_dir=$VOLUME_DYNASEQDESTDIR +chmod -R 777 $VOLUME_DYNASEQDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --tensorrt --variable --models_dir=$VOLUME_VARSEQDESTDIR +chmod -R 777 $VOLUME_VARSEQDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_dyna_sequence_implicit_models.py --tensorrt --models_dir=$VOLUME_DYNASEQIMPLICITDESTDIR +chmod -R 777 $VOLUME_DYNASEQIMPLICITDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_ragged_models.py --tensorrt --models_dir=$VOLUME_RAGGEDDESTDIR +chmod -R 777 $VOLUME_RAGGEDDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_trt_format_models.py --models_dir=$VOLUME_FORMATDESTDIR +chmod -R 777 $VOLUME_FORMATDESTDIR +python3 $VOLUME_SRCDIR/gen_qa_trt_data_dependent_shape.py --models_dir=$VOLUME_DATADEPENDENTDIR +chmod -R 777 $VOLUME_DATADEPENDENTDIR +# Make shared library for custom Hardmax plugin. +if [ -d "/usr/src/tensorrt" ]; then + cd /usr/src/tensorrt/samples/python/onnx_custom_plugin else - echo "[WARNING] TensorRT release/\${TRT_BRANCH} not available on github.com/NVIDIA/TensorRT; skipping CustomHardmax plugin model generation. L0_trt_plugin coverage will be missing for this TRT version." + git clone -b release/${TENSORRT_VERSION} https://github.com/NVIDIA/TensorRT.git + cd /workspace/TensorRT/samples/python/onnx_custom_plugin fi -exit 0 +rm -rf build && mkdir build && \ +cd build && cmake .. && make -j && cp libcustomHardmaxPlugin.so $VOLUME_PLGDESTDIR/. +LD_PRELOAD=$VOLUME_PLGDESTDIR/libcustomHardmaxPlugin.so python3 $VOLUME_SRCDIR/gen_qa_trt_plugin_models.py --models_dir=$VOLUME_PLGDESTDIR +chmod -R 777 $VOLUME_PLGDESTDIR EOF - log_message.status "run: chmod a+x ${TRTSCRIPT}" - chmod a+x ${TRTSCRIPT} - if [ $? -ne 0 ]; then - log_message.error "failed: chmod ${TRTSCRIPT}" - exit 1 - fi - -} - -log_message.status "check: engine installation" -if [ "$TRITON_MODELS_USE_DOCKER" -eq 1 ] && which docker ; then - log_message.info "Docker is installed." - - define_model_output_directories - - SCRIPT_NAME_SUFFIX=docker.v2 define_models_generation_scripts - - if ! docker volume inspect $DOCKER_VOLUME > /dev/null 2>&1; then - log_message.status "docker volume: $DOCKER_VOLUME does not exist. Creating..." - docker volume create $DOCKER_VOLUME --label RUNNER_ID=$RUNNER_ID --label PROJECT_NAME=$PROJECT_NAME - log_message.status "docker volume: $DOCKER_VOLUME created" - docker volume inspect $DOCKER_VOLUME - else - log_message.status "docker volume: $DOCKER_VOLUME in use" - docker volume inspect $DOCKER_VOLUME - fi - - log_message.status "docker pull: $UBUNTU_IMAGE" - docker pull $UBUNTU_IMAGE - - log_message.status "docker volume: create destination directory on volume" - log_message.info "docker run -v $DOCKER_VOLUME:/mnt -w /mnt/$CI_JOB_ID $UBUNTU_IMAGE mkdir -p gen_srcdir ${TRITON_VERSION}" - docker run \ - --rm \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - -v $DOCKER_VOLUME:/mnt \ - -w /mnt/$CI_JOB_ID \ - $UBUNTU_IMAGE \ - mkdir -p gen_srcdir ${TRITON_VERSION} - - log_message.status "docker volume: create model directories on volume" - docker run \ - --rm \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - -v $DOCKER_VOLUME:/mnt \ - -w /mnt/$CI_JOB_ID \ - $UBUNTU_IMAGE \ - mkdir -p \ - $TRITON_MDLS_BLD_DIR \ - $TRITON_MDLS_SRC_DIR \ - $TRITON_MDLS_QA_MODEL \ - $TRITON_MDLS_QA_VARIABLE_MODEL \ - $TRITON_MDLS_QA_IDENTITY_MODEL \ - $TRITON_MDLS_QA_IDENTITY_BIG_MODEL \ - $TRITON_MDLS_QA_SHAPETENSOR_MODEL \ - $TRITON_MDLS_QA_RESHAPE_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_ENSEMBLE_MODEL \ - $TRITON_MDLS_QA_NOSHAPE_MODEL \ - $TRITON_MDLS_QA_TRT_PLUGIN_MODEL \ - $TRITON_MDLS_QA_RAGGED_MODEL \ - $TRITON_MDLS_QA_TRT_FORMAT_MODEL \ - $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_TORCHTRT_MODEL \ - $TRITON_MDLS_QA_SCALAR_MODELS \ - $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL - - log_message.status "docker container: create container $DOCKER_VOLUME_CONTAINER" - log_message.info "docker create --name $DOCKER_VOLUME_CONTAINER -v $DOCKER_VOLUME:/mnt -w /mnt/$CI_JOB_ID $UBUNTU_IMAGE" - docker create \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - --name $DOCKER_VOLUME_CONTAINER \ - -v $DOCKER_VOLUME:/mnt \ - -w /mnt/$CI_JOB_ID \ - $UBUNTU_IMAGE - - log_message.status "docker container: copy script to container" - docker cp . $DOCKER_VOLUME_CONTAINER:/mnt/$CI_JOB_ID/gen_srcdir - - if [[ "aarch64" != $(uname -m) ]] ; then - - log_message.status "docker run: $OPENVINOSCRIPT" - log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $UBUNTU_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT" - docker run \ - --rm \ - -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - -t \ - $UBUNTU_IMAGE \ - bash -e $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT - - exit_code=$? - - if [ $exit_code -ne 0 ]; then - log_message.error "docker run: ${OPENVINOSCRIPT} failed" - exit 1 - fi - - rm $OPENVINOSCRIPT - fi # [[ "aarch64" != $(uname -m) ]] - - log_message.status "docker run: $ONNXSCRIPT" - log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $UBUNTU_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT" - docker run \ - --rm \ - -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - -t \ - $UBUNTU_IMAGE \ - bash -e $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT - - exit_code=$? - - if [ $exit_code -ne 0 ]; then - log_message.error "docker run: ${ONNXSCRIPT} failed" - exit 1 - fi - - rm $ONNXSCRIPT - - log_message.status "docker pull: $PYTORCH_IMAGE" - log_message.info "docker pull $PYTORCH_IMAGE" - docker pull $PYTORCH_IMAGE - - log_message.status "docker run: $TORCHSCRIPT" - log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $PYTORCH_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT" - docker run \ - --rm \ - -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - -t \ - $PYTORCH_IMAGE \ - bash -e $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT - - exit_code=$? - - if [ $exit_code -ne 0 ]; then - log_message.error "docker run: ${TORCHSCRIPT} failed" - exit 1 - fi - - rm $TORCHSCRIPT - - if [ "$MODEL_TYPE" != "igpu" ] ; then - log_message.status "docker pull: $TENSORRT_IMAGE" - docker pull $TENSORRT_IMAGE - - log_message.status "docker run: $TRTSCRIPT" - log_message.info "docker run $DOCKER_GPU_ARGS -v $DOCKER_VOLUME:/mnt $TENSORRT_IMAGE bash -e $TRITON_MDLS_SRC_DIR/$TRTSCRIPT" - docker run \ - --rm \ - -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - -t \ - -e TRT_VERBOSE \ - $TENSORRT_IMAGE \ - bash -e $TRITON_MDLS_SRC_DIR/$TRTSCRIPT - - exit_code=$? - - if [ $exit_code -ne 0 ]; then - log_message.error "docker run: ${TRTSCRIPT} failed" - exit 1 - fi - - rm $TRTSCRIPT - fi # [ "$MODEL_TYPE" != "igpu" ] - - if [ -z $CI ] ; then - log_message.status "docker cp:copying generated models to /tmp/" - docker cp $DOCKER_VOLUME_CONTAINER:$TRITON_MDLS_BLD_DIR/$TRITON_VERSION /tmp/ - log_message.status "docker rm: removing docker container $DOCKER_VOLUME_CONTAINER" - docker rm -f $(docker ps -a --filter volume=$DOCKER_VOLUME --format '{{ .ID }}') - log_message.status "docker volume rm: removing docker volume $DOCKER_VOLUME" - docker volume rm $DOCKER_VOLUME - fi # [ -z $CI ] - -elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then - log_message.info "NVIDIA Enroot is installed." ; - - TRITON_MDLS_BLD_DIR="/tmp/$CI_JOB_ID" define_model_output_directories - - SCRIPT_NAME_SUFFIX=enroot.v1 define_models_generation_scripts - - log_message.status "cleanup models folder if exists: $TRITON_MDLS_BLD_DIR" - rm -rf $TRITON_MDLS_BLD_DIR - - log_message.status "create models directory structure in: $TRITON_MDLS_BLD_DIR" - mkdir -p \ - $TRITON_MDLS_BLD_DIR \ - $TRITON_MDLS_SRC_DIR \ - $TRITON_MDLS_QA_MODEL \ - $TRITON_MDLS_QA_VARIABLE_MODEL \ - $TRITON_MDLS_QA_IDENTITY_MODEL \ - $TRITON_MDLS_QA_IDENTITY_BIG_MODEL \ - $TRITON_MDLS_QA_SHAPETENSOR_MODEL \ - $TRITON_MDLS_QA_RESHAPE_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_DYNA_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_DYNA_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_MODEL \ - $TRITON_MDLS_QA_ENSEMBLE_MODEL \ - $TRITON_MDLS_QA_NOSHAPE_MODEL \ - $TRITON_MDLS_QA_TRT_PLUGIN_MODEL \ - $TRITON_MDLS_QA_RAGGED_MODEL \ - $TRITON_MDLS_QA_TRT_FORMAT_MODEL \ - $TRITON_MDLS_QA_TRT_DATA_DEPENDENT_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_VARIABLE_SEQUENCE_INITIAL_STATE_IMPLICIT_MODEL \ - $TRITON_MDLS_QA_TORCHTRT_MODEL \ - $TRITON_MDLS_QA_SCALAR_MODELS \ - $TRITON_MDLS_QA_DYNAMIC_BATCH_IMAGE_MODEL - - log_message.status "copy scripts to: $TRITON_MDLS_SRC_DIR" - cp -rv $TRITON_MDLS_BASE_SCRIPT_DIR/* $TRITON_MDLS_SRC_DIR/ - - log_message.status "enroot import: $UBUNTU_IMAGE to ubuntu.$CI_JOB_ID.enroot.sqsh" - enroot import --output /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh docker://$UBUNTU_IMAGE - - - log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" - enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh - log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT - if [ $? -ne 0 ]; then - log_message.error "enroot start: ${OPENVINOSCRIPT} failed" - exit 1 - fi - - - log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" - enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh - log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT - if [ $? -ne 0 ]; then - log_message.error "enroot start: ${ONNXSCRIPT} failed" - exit 1 - fi - - - log_message.status "enroot import: $PYTORCH_IMAGE to /tmp/pytorch.$CI_JOB_ID.enroot.sqsh" - enroot import --output /tmp/pytorch.$CI_JOB_ID.enroot.sqsh docker://$PYTORCH_IMAGE - if [ $? -ne 0 ]; then - log_message.error "enroot import: ${PYTORCH_IMAGE} failed" - exit 1 - fi - - log_message.status "enroot create: pytorch.$CI_JOB_ID" - enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh - log_message.info "enroot start: pytorch.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT - if [ $? -ne 0 ]; then - log_message.error "enroot start: ${TORCHSCRIPT} failed" - exit 1 - fi - - log_message.status "enroot import: $TENSORRT_IMAGE to /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh" - enroot import --output /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh docker://$TENSORRT_IMAGE - log_message.status "enroot create: tensorrt.$CI_JOB_ID" - enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh - log_message.info "enroot start: tensorrt.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT - if [ $? -ne 0 ]; then - log_message.error "enroot start: ${TRTSCRIPT} failed" - exit 1 - fi +chmod a+x $TRTSCRIPT +if [ $? -ne 0 ]; then + echo -e "Failed: chmod" + exit 1 +fi + +if [ "$MODEL_TYPE" != "igpu" ] ; then + docker cp $TRTSCRIPT $DOCKER_VOLUME_CONTAINER:$VOLUME_SRCDIR + docker pull $TENSORRT_IMAGE + + echo -e "\033[34m[ INFO ] - Running: $TRTSCRIPT \033[0m " + + docker run \ + --rm \ + --label RUNNER_ID=$RUNNER_ID \ + --label PROJECT_NAME=$PROJECT_NAME \ + $DOCKER_GPU_ARGS \ + -v $DOCKER_VOLUME:/mnt \ + $TENSORRT_IMAGE \ + bash -xe $VOLUME_SRCDIR/$TRTSCRIPT + + if [ $? -ne 0 ]; then + echo -e "Failed" + exit 1 + fi -else - log_message.warning "Neither Docker nor NVIDIA Enroot is installed." ; - log_message.warning "Please install Docker or NVIDIA Enroot to generate the models." ; fi + +if [ -z $CI ] ; then + echo -e "\033[34m[ INFO ] - Copying generated models to /tmp/ \033[0m " + docker cp $DOCKER_VOLUME_CONTAINER:$VOLUME_BUILD_DIR/$TRITON_VERSION /tmp/ + echo -e "\033[34m[ INFO ] - Removing Docker container $DOCKER_VOLUME_CONTAINER \033[0m " + docker rm -f $(docker ps -a --filter volume=$DOCKER_VOLUME --format '{{ .ID }}') + echo -e "\033[34m[ INFO ] - Removing Docker volume $DOCKER_VOLUME \033[0m " + docker volume rm $DOCKER_VOLUME +fi \ No newline at end of file diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index d305f07ce3..cd7efea723 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,7 +28,6 @@ import argparse import os -import sys from builtins import range import gen_ensemble_model_utils as emu @@ -46,14 +45,6 @@ np_dtype_string = np.dtype(object) from typing import List, Tuple -_color_blue = "\033[94m" -_color_cyan = "\033[36m" -_color_green = "\033[32m" -_color_magenta = "\033[35m" -_color_red = "\033[31m" -_color_reset = "\033[0m" -_color_yellow = "\033[33m" - def create_plan_dynamic_rf_modelfile( models_dir, @@ -75,11 +66,7 @@ def create_plan_dynamic_rf_modelfile( trt_memory_format = trt.TensorFormat.LINEAR # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() if max_batch == 0: @@ -160,8 +147,7 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) - if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): - flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) + flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: @@ -194,7 +180,7 @@ def create_plan_dynamic_rf_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -220,11 +206,7 @@ def create_plan_dynamic_modelfile( trt_output1_dtype = np_to_trt_dtype(output1_dtype) # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() if max_batch == 0: @@ -358,7 +340,6 @@ def create_plan_dynamic_modelfile( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating model {model_name}{_color_reset}") if min_dim != 1 or max_dim != 32: model_name = "{}-{}-{}".format(model_name, min_dim, max_dim) @@ -366,7 +347,7 @@ def create_plan_dynamic_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -391,11 +372,7 @@ def create_plan_fixed_rf_modelfile( trt_memory_format = trt.TensorFormat.LINEAR # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() if max_batch == 0: @@ -451,8 +428,7 @@ def create_plan_fixed_rf_modelfile( profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) - if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): - flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) + flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: @@ -479,12 +455,11 @@ def create_plan_fixed_rf_modelfile( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -508,11 +483,7 @@ def create_plan_fixed_modelfile( trt_output1_dtype = np_to_trt_dtype(output1_dtype) # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() if max_batch == 0: @@ -567,12 +538,11 @@ def create_plan_fixed_modelfile( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -597,6 +567,9 @@ def create_plan_modelfile( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return @@ -713,6 +686,9 @@ def create_plan_modelconfig( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return @@ -734,7 +710,6 @@ def create_plan_modelconfig( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating config for {model_name}{_color_reset}") if min_dim != 1 or max_dim != 32: model_name = "{}-{}-{}".format(model_name, min_dim, max_dim) @@ -840,7 +815,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -867,6 +842,9 @@ def create_onnx_modelfile( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return @@ -885,7 +863,6 @@ def create_onnx_modelfile( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) batch_dim = [] if max_batch == 0 else [None] @@ -955,7 +932,7 @@ def create_onnx_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") @@ -964,6 +941,7 @@ def create_onnx_modelfile( def create_onnx_modelconfig( models_dir, max_batch, + model_version, input_shape, output0_shape, output1_shape, @@ -977,6 +955,9 @@ def create_onnx_modelconfig( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return @@ -987,9 +968,6 @@ def create_onnx_modelconfig( output0_dtype, output1_dtype, ) - - print(f"{_color_green}Creating config for {model_name}{_color_reset}") - config_dir = models_dir + "/" + model_name # [TODO] move create_general_modelconfig() out of emu as it is general @@ -1011,15 +989,15 @@ def create_onnx_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir - with open(config_dir + "/config.pbtxt", "w") as file: - file.write(config) + with open(config_dir + "/config.pbtxt", "w") as cfile: + cfile.write(config) - with open(config_dir + "/output0_labels.txt", "w") as file: + with open(config_dir + "/output0_labels.txt", "w") as lfile: for l in range(output0_label_cnt): - file.write("label" + str(l) + "\n") + lfile.write("label" + str(l) + "\n") def create_libtorch_modelfile( @@ -1054,9 +1032,6 @@ def create_libtorch_modelfile( output0_dtype, output1_dtype, ) - - print(f"{_color_green}Creating model {model_name}{_color_reset}") - # handle for -1 (when variable) since can't create tensor with shape of [-1] input_shape = [abs(ips) for ips in input_shape] @@ -1282,367 +1257,20 @@ def forward(self, INPUT0, INPUT1): addSubModel = AddSubNet((torch_output0_dtype, torch_output1_dtype, swap)) traced = torch.jit.script(addSubModel) - model_version_dir = f"{models_dir}/{model_name}/{model_version}" - - try: - os.makedirs(model_version_dir) - except OSError: - pass # ignore existing dir - - traced.save(f"{model_version_dir}/model.pt") - - -def generate_torch_aoti_sample_inputs( - input_shape, - input_dtype, - device, -): - # handle for -1 (when variable) since can't create tensor with shape of [-1] - input_shape = [abs(ips) for ips in input_shape] - - np_to_torch_dtype = { - np.int8: torch.int8, - np.int16: torch.int16, - np.int32: torch.int32, - np.int64: torch.int64, - np.float16: torch.float16, - np.float32: torch.float32, - np.float64: torch.float64, - np.uint8: torch.uint8, - np.uint16: torch.uint16, - np.uint32: torch.uint32, - np.uint64: torch.uint64, - } - - if input_dtype not in np_to_torch_dtype: - print( - f"{_color_yellow}warning: dtype {input_dtype} is unsupported; falling back to torch.int32{_color_reset}" - ) - input_dtype = np.int32 - - input0 = torch.zeros( - input_shape, dtype=np_to_torch_dtype[input_dtype], device=device - ) - input1 = torch.zeros( - input_shape, dtype=np_to_torch_dtype[input_dtype], device=device - ) - - return (input0, input1) - - -def np_to_dtype(np_dtype): - if np_dtype == np.int8: - return torch.int8 - elif np_dtype == np.int16: - return torch.int16 - elif np_dtype == np.int32: - return torch.int32 - elif np_dtype == np.int64: - return torch.int64 - elif np_dtype == np.float16: - return torch.float16 - elif np_dtype == np.float32: - return torch.float32 - elif np_dtype == np.float64: - return torch.float64 - elif np_dtype == np.uint8: - return torch.uint8 - elif np_dtype == np.uint16: - return torch.uint16 - elif np_dtype == np.uint32: - return torch.uint32 - elif np_dtype == np.uint64: - return torch.uint64 - else: - print( - f"{_color_yellow}warning: dtype {np_dtype} is unsupported; falling back to torch.int32{_color_reset}" - ) - return torch.int32 - - -def create_torch_aoti_model_file( - models_dir, - model_version, - input_shape, - input_dtype, - output_dtype, - swap=False, -): - model_name = tu.get_model_name( - "torch_aoti", - input_dtype, - output_dtype, - None, - ) - - # AOTI compiles tensor operations and does not support string (object) dtype - if input_dtype == np_dtype_string or output_dtype == np_dtype_string: - print( - f"{_color_yellow}warning: Skipping AOTI model {model_name}: " - f"string/object dtype is not supported for AOTI compilation{_color_reset}" - ) - return False - - model_version_dir = os.path.join(models_dir, model_name, str(model_version)) - - print(f"{_color_green}Creating model {model_name}{_color_reset}") - - torch_input_dtype: torch.dtype = np_to_dtype(input_dtype) - torch_output_dtype: torch.dtype = np_to_dtype(output_dtype) - - print(f"{model_name}({torch_input_dtype}) -> {torch_output_dtype}") - - # handle for -1 (when variable) since can't create tensor with shape of [-1] - input_shape = [abs(ips) for ips in input_shape] - - try: - os.makedirs(model_version_dir) - except OSError: - pass # ignore existing dir - - class AddSubNet(nn.Module): - def __init__( - self, - swap: bool, - input_dtype: torch.dtype, - output_dtype: torch.dtype, - ) -> None: - self.swap = swap - self.input_dtype = input_dtype - self.output_dtype = output_dtype - super(AddSubNet, self).__init__() - - def forward(self, INPUT0: torch.Tensor, INPUT1: torch.Tensor) -> torch.Tensor: - if INPUT0.dtype != self.input_dtype: - raise TypeError( - f"INPUT0 expected {self.input_dtype} vs. actual {INPUT0.dtype} type." - ) - if INPUT1.dtype != self.input_dtype: - raise TypeError( - f"INPUT1 expected {self.input_dtype} vs. actual {INPUT1.dtype} type." - ) - return (INPUT0 - INPUT1 if self.swap else INPUT0 + INPUT1).to( - self.output_dtype, - ) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = AddSubNet(swap, torch_input_dtype, torch_output_dtype) - model.to(device) - model = model.eval() - - sample_inputs = generate_torch_aoti_sample_inputs(input_shape, input_dtype, device) - package_path = os.path.join(model_version_dir, "model.pt2") - - try: - exported_model = torch.export.export(model, sample_inputs) - torch._inductor.aoti_compile_and_package( - exported_model, - package_path=package_path, - ) - except Exception as e: - print( - f"{_color_red}error: Failed to create model {model_name}{_color_reset}", - file=sys.stderr, - ) - print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) - return False - - return True - - -def create_torch_aoti_complex_model_file( - models_dir: str, -): - base_name = "torch_aoti_complex" - model_names = [ - f"{base_name}_named", - f"{base_name}_index", - ] - model_version_dirs = [ - os.path.join(models_dir, model_names[0], "1"), - os.path.join(models_dir, model_names[1], "1"), - ] - - for model_version_dir in model_version_dirs: - try: - os.makedirs(model_version_dir) - except OSError: - pass # ignore existing dir - - print(f"{_color_green}Creating model {base_name}{_color_reset}") - - class TorchAotiComplex(torch.nn.Module): - def __init__(self): - super().__init__() - - def forward( - self, - hdata: torch.Tensor, - vdata: torch.Tensor, - options: dict[str, torch.Tensor], - ) -> dict[ - str, - torch.Tensor | tuple[torch.Tensor, torch.Tensor] | dict[str, torch.Tensor], - ]: - out = { - "AAA": hdata + vdata, - "ZZZ": hdata - vdata, - "BBB": ( - hdata, - vdata, - ), - "CCC": options, - } - - return out - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = TorchAotiComplex() - model.to(device) - model = model.eval() - - SHAPE = (1, 16) - - sample_args = ( - torch.zeros(SHAPE, dtype=torch.int8, device=device), - torch.zeros(SHAPE, dtype=torch.int8, device=device), - { - "option1": torch.zeros(SHAPE, dtype=torch.int8, device=device), - "option2": torch.zeros(SHAPE, dtype=torch.int8, device=device), - }, - ) - - # Export and package the model - print(f"{_color_green}Exporting and packaging the model...{_color_reset}") - - model_file_name = "model.pt2" - package_paths = [ - os.path.join(model_version_dirs[0], model_file_name), - os.path.join(model_version_dirs[1], model_file_name), - ] - - try: - exported_model = torch.export.export(model, sample_args) - torch._inductor.aoti_compile_and_package( - exported_model, - package_path=package_paths[0], - ) - except Exception as e: - print( - f"{_color_red}error: Failed to create model {base_name}{_color_reset}", - file=sys.stderr, - ) - print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) - return False - - try: - # Now load and run the packaged model - print(f"{_color_cyan}Loading and running the packaged model...{_color_reset}") - - compiled_model = torch._inductor.aoti_load_package(package_paths[0]) - - print(f"{_color_cyan}Compiled model call spec:{_color_reset}") - - for elem in compiled_model.loader.get_call_spec(): - print(elem) - - print(f"{_color_cyan}Running the compiled model...{_color_reset}") - - with torch.inference_mode(): - hdata = torch.randint( - low=0, - high=127, - size=SHAPE, - dtype=torch.int8, - device=device, - ) - vdata = torch.randint( - low=0, - high=127, - size=SHAPE, - dtype=torch.int8, - device=device, - ) - options = { - "option1": torch.randint( - low=0, - high=127, - size=SHAPE, - dtype=torch.int8, - device=device, - ), - "option2": torch.randint( - low=0, - high=127, - size=SHAPE, - dtype=torch.int8, - device=device, - ), - } - - _ = compiled_model(hdata, vdata, options) - - print( - f'{_color_green}Model "{base_name}" successfully executed.{_color_reset}' - ) - except Exception as e: - print( - f"{_color_red}error: Failed to validate model {base_name}{_color_reset}", - file=sys.stderr, - ) - print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) - return False - - # Copy the compiled model package to the alternate model folder. - # Both the named and ordinal addressing versions of the model (from Triton's point-of-view) use the same compiled model. - shutil.copy(package_paths[0], package_paths[1]) - - return True - - -def create_torchvision_aoti_model_file( - models_dir: str, - max_batch: int, -): - model_name = "torchvision_aoti" - model_version_dir = os.path.join(models_dir, model_name, "1") + model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir - print(f"{_color_green}Creating model {model_name}{_color_reset}") - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = models.resnet50(weights=models.ResNet50_Weights.DEFAULT) - model = model.to(device) - model = model.eval() - - SHAPE = (max_batch, 3, 224, 224) - - # Example input tensor with batch size 1 and 3 color channels (RGB), height and width of 224 - sample_inputs = (torch.zeros(SHAPE, dtype=torch.float32, device=device),) - - package_path = os.path.join(model_version_dir, "model.pt2") - - try: - ep = torch.export.export(model, sample_inputs) - torch._inductor.aoti_compile_and_package(ep, package_path=package_path) - except Exception as e: - print( - f"{_color_red}error: Failed to create model {model_name}{_color_reset}", - file=sys.stderr, - ) - print(f"\n{_color_red}{e}{_color_reset}\n", file=sys.stderr) - return False - - return True + traced.save(model_version_dir + "/model.pt") def create_libtorch_modelconfig( models_dir, max_batch, + model_version, input_shape, output0_shape, output1_shape, @@ -1668,9 +1296,9 @@ def create_libtorch_modelconfig( if version_policy is not None: type, val = version_policy if type == "latest": - version_policy_str = f"{{ latest {{ num_versions: {val} }} }}" + version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(val) elif type == "specific": - version_policy_str = f"{{ specific {{ versions: {val} }} }}" + version_policy_str = "{{ specific {{ versions: {} }}}}".format(val) else: version_policy_str = "{ all { }}" @@ -1681,342 +1309,62 @@ def create_libtorch_modelconfig( output0_dtype, output1_dtype, ) - - print(f"{_color_green}Creating config for {model_name}{_color_reset}") - - label_filename = "output0_labels.txt" - config_dir = f"{models_dir}/{model_name}" - config = f""" -backend: "pytorch" -name: "{model_name}" + config_dir = models_dir + "/" + model_name + config = """ +name: "{}" platform: "pytorch_libtorch" -max_batch_size: {max_batch} -version_policy: {version_policy_str} +max_batch_size: {} +version_policy: {} input [ {{ name: "INPUT0" - data_type: {np_to_model_dtype(input_dtype)} - dims: [ {tu.shape_to_dims_str(input_shape)} ] + data_type: {} + dims: [ {} ] }}, {{ name: "INPUT1" - data_type: {np_to_model_dtype(input_dtype)} - dims: [ {tu.shape_to_dims_str(input_shape)} ] + data_type: {} + dims: [ {} ] }} ] output [ {{ name: "OUTPUT__0" - data_type: {np_to_model_dtype(output0_dtype)} - dims: [ {tu.shape_to_dims_str(output0_shape)} ] - label_filename: "{label_filename}" + data_type: {} + dims: [ {} ] + label_filename: "output0_labels.txt" }}, {{ name: "OUTPUT__1" - data_type: {np_to_model_dtype(output1_dtype)} - dims: [ {tu.shape_to_dims_str(output1_shape)} ] + data_type: {} + dims: [ {} ] }} ] -""" - - try: - os.makedirs(config_dir) - except OSError: - pass # ignore existing dir - - config_path = os.path.join(config_dir, "config.pbtxt") - - with open(config_path, "w") as file: - file.write(config) - print(f"Created {config_path}") - - with open(f"{config_dir}/{label_filename}", "w") as file: - for l in range(output0_label_cnt): - file.write("label" + str(l) + "\n") - print(f"Created {config_dir}/{label_filename}") - - -def create_torch_aoti_model_config( - models_dir, - input_shape, - output_shape, - input_dtype, - output_dtype, - output_label_cnt, - version_policy, -): - # Unpack version policy - version_policy_str = "{ latest { num_versions: 1 }}" - if version_policy is not None: - type, val = version_policy - if type == "latest": - version_policy_str = f"{{ latest {{ num_versions: {val} }} }}" - elif type == "specific": - version_policy_str = f"{{ specific {{ versions: {val} }} }}" - else: - version_policy_str = "{ all { }}" - - # Use a different model name for the non-batching variant - model_name = tu.get_model_name( - "torch_aoti", - input_dtype, - output_dtype, - None, +""".format( + model_name, + max_batch, + version_policy_str, + np_to_model_dtype(input_dtype), + tu.shape_to_dims_str(input_shape), + np_to_model_dtype(input_dtype), + tu.shape_to_dims_str(input_shape), + np_to_model_dtype(output0_dtype), + tu.shape_to_dims_str(output0_shape), + np_to_model_dtype(output1_dtype), + tu.shape_to_dims_str(output1_shape), ) - print(f"{_color_green}Creating config for {model_name}{_color_reset}") - - label_filename = "output_labels.txt" - config_dir = os.path.join(models_dir, model_name) - config = f""" -backend: "pytorch" -name: "{model_name}" -platform: "torch_aoti" -version_policy: {version_policy_str} -input [ - {{ - name: "ARGS[0]" - data_type: {np_to_model_dtype(input_dtype)} - dims: [ {tu.shape_to_dims_str(input_shape)} ] - }}, - {{ - name: "ARGS[1]" - data_type: {np_to_model_dtype(input_dtype)} - dims: [ {tu.shape_to_dims_str(input_shape)} ] - }} -] -output [ - {{ - name: "RESULT" - data_type: {np_to_model_dtype(output_dtype)} - dims: [ {tu.shape_to_dims_str(output_shape)} ] - label_filename: "{label_filename}" - }} -] -instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] -""" - - try: - os.makedirs(config_dir) - except OSError: - pass # ignore existing dir - - config_path = os.path.join(config_dir, "config.pbtxt") - - with open(config_path, "w") as file: - file.write(config) - print(f"Created {config_path}") - - label_path = os.path.join(config_dir, label_filename) - - with open(label_path, "w") as file: - for l in range(output_label_cnt): - file.write(f"label{l}\n") - print(f"Created {label_path}") - - -def create_torch_aoti_complex_model_config( - models_dir, -): - base_name = "torch_aoti_complex" - model_names = [ - f"{base_name}_named", - f"{base_name}_index", - ] - - print(f"{_color_green}Creating config for {base_name}{_color_reset}") - - config_dirs = [ - os.path.join(models_dir, model_names[0]), - os.path.join(models_dir, model_names[1]), - ] - configs = [ - f""" -backend: "pytorch" -platform: "torch_aoti" -name: "{model_names[0]}" -input: [ - {{ - name: "ARGS[0]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "ARGS[1]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "ARGS[2][option1]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "ARGS[2][option2]" - data_type: TYPE_INT8 - dims: [1, 16] - }} -] -output: [ - {{ - name: "RESULT[AAA]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "RESULT[BBB][0]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "RESULT[BBB][1]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "RESULT[CCC][option1]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "RESULT[CCC][option2]" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "RESULT[ZZZ]" - data_type: TYPE_INT8 - dims: [1, 16] - }} -] -instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] -""", - f""" -backend: "pytorch" -name: "{model_names[1]}" -platform: "torch_aoti" -input: [ - {{ - name: "INPUT__0" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "INPUT__1" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "INPUT__2" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "INPUT__3" - data_type: TYPE_INT8 - dims: [1, 16] - }} -] -output: [ - {{ - name: "OUTPUT__0" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "OUTPUT__1" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "OUTPUT__2" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "OUTPUT__3" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "OUTPUT__4" - data_type: TYPE_INT8 - dims: [1, 16] - }}, - {{ - name: "OUTPUT__5" - data_type: TYPE_INT8 - dims: [1, 16] - }} -] -instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] -""", - ] - - for i in range(2): - config_dir = config_dirs[i] - try: - os.makedirs(config_dir) - except OSError: - pass # ignore existing dir - - config_path = os.path.join(config_dir, "config.pbtxt") - - with open(config_path, "w") as file: - file.write(configs[i]) - print(f"Created {config_path}") - - -def create_torchvision_aoti_model_config( - models_dir: str, - max_batch: int, -): - model_name = "torchvision_aoti" - label_filename = "resnet50_labels.txt" - - print(f"{_color_green}Creating config for {model_name}{_color_reset}") - - config_dir = os.path.join(models_dir, model_name) - config = f""" -backend: "pytorch" -name: "{model_name}" -platform: "torch_aoti" -max_batch_size: {max_batch} -input [ - {{ - name: "ARGS[0]" - data_type: TYPE_FP32 - dims: [ 3, 224, 224 ] - }}] -output [ - {{ - name: "RESULT" - data_type: TYPE_FP32 - dims: [ 1000 ] - label_filename: "{label_filename}" - }} -] -instance_group [{{ kind: {"KIND_GPU" if torch.cuda.is_available() else "KIND_CPU"} }}] -""" - try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir - config_path = os.path.join(config_dir, "config.pbtxt") - - with open(config_path, "w") as file: - file.write(config) - print(f"Created {config_path}") - - source_path = os.environ.get("TRITON_GENSRCDIR", default="gen_srcdir") - source_filename = os.path.join(source_path, RESNET50_LABEL_FILE) - - target_path = os.path.join(config_dir, label_filename) + with open(config_dir + "/config.pbtxt", "w") as cfile: + cfile.write(config) - shutil.copyfile(source_filename, target_path) - print(f"Created {target_path}") + with open(config_dir + "/output0_labels.txt", "w") as lfile: + for l in range(output0_label_cnt): + lfile.write("label" + str(l) + "\n") def create_openvino_modelfile( @@ -2037,6 +1385,8 @@ def create_openvino_modelfile( output0_dtype, output1_dtype, batch_dim + input_shape, + batch_dim + output0_shape, + batch_dim + output1_shape, ): return @@ -2047,7 +1397,6 @@ def create_openvino_modelfile( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating model {model_name}{_color_reset}") model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) in0 = ov.opset1.parameter( @@ -2073,6 +1422,7 @@ def create_openvino_modelfile( def create_openvino_modelconfig( models_dir, max_batch, + model_version, input_shape, output0_shape, output1_shape, @@ -2088,6 +1438,8 @@ def create_openvino_modelconfig( output0_dtype, output1_dtype, batch_dim + input_shape, + batch_dim + output0_shape, + batch_dim + output1_shape, ): return @@ -2109,7 +1461,6 @@ def create_openvino_modelconfig( output0_dtype, output1_dtype, ) - print(f"{_color_green}Creating config for {model_name}{_color_reset}") config_dir = models_dir + "/" + model_name # platform is empty and backend is 'openvino' for openvino model @@ -2159,7 +1510,7 @@ def create_openvino_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -2181,10 +1532,8 @@ def create_models( output0_label_cnt, version_policy=None, ): - print(f"{_color_blue}Creating models in {models_dir}{_color_reset}") model_version = 1 if FLAGS.tensorrt: - print(f"{_color_magenta}TensorRT model generation requested{_color_reset}") # max-batch 8 suffix = () if ( @@ -2275,11 +1624,11 @@ def create_models( ) if FLAGS.onnx: - print(f"{_color_magenta}ONNX model generation requested{_color_reset}") # max-batch 8 create_onnx_modelconfig( models_dir, 8, + model_version, input_shape, output0_shape, output1_shape, @@ -2304,6 +1653,7 @@ def create_models( create_onnx_modelconfig( models_dir, 0, + model_version, input_shape, output0_shape, output1_shape, @@ -2326,11 +1676,11 @@ def create_models( ) if FLAGS.libtorch: - print(f"{_color_magenta}PyTorch: PT model generation requested{_color_reset}") # max-batch 8 create_libtorch_modelconfig( models_dir, 8, + model_version, input_shape, output0_shape, output1_shape, @@ -2355,6 +1705,7 @@ def create_models( create_libtorch_modelconfig( models_dir, 0, + model_version, input_shape, output0_shape, output1_shape, @@ -2376,35 +1727,12 @@ def create_models( output1_dtype, ) - if FLAGS.torch_aoti: - if output0_dtype == output1_dtype: - print( - f"{_color_magenta}PyTorch: AOTI model generation requested{_color_reset}" - ) - # max-batch 8 - if create_torch_aoti_model_file( - models_dir, - model_version, - input_shape, - input_dtype, - output0_dtype, - ): - create_torch_aoti_model_config( - models_dir, - input_shape, - output0_shape, - input_dtype, - output0_dtype, - output0_label_cnt, - version_policy, - ) - if FLAGS.openvino: - print(f"{_color_magenta}OpenVINO model generation requested{_color_reset}") # max-batch 8 create_openvino_modelconfig( models_dir, 8, + model_version, input_shape, output0_shape, output1_shape, @@ -2429,6 +1757,7 @@ def create_models( create_openvino_modelconfig( models_dir, 0, + model_version, input_shape, output0_shape, output1_shape, @@ -2451,7 +1780,6 @@ def create_models( ) if FLAGS.ensemble: - print(f"{_color_magenta}Ensemble model generation requested{_color_reset}") for pair in emu.platform_types_and_validation(): if not pair[1]( input_dtype, @@ -2589,18 +1917,6 @@ def create_fixed_models( action="store_true", help="Generate Pytorch LibTorch models", ) - parser.add_argument( - "--torch-aoti", - required=False, - action="store_true", - help="Generate Pytorch LibTorch models using PT2", - ) - parser.add_argument( - "--torchvision-aoti", - required=False, - action="store_true", - help="Generate Pytorch Torchvision models using PT2", - ) parser.add_argument( "--openvino", required=False, @@ -2627,18 +1943,9 @@ def create_fixed_models( import tensorrt as trt if FLAGS.onnx: import onnx - if FLAGS.libtorch or FLAGS.torch_aoti: - import shutil - + if FLAGS.libtorch: import torch from torch import nn - if FLAGS.torchvision_aoti: - import shutil - - import torch - import torchvision.models as models - - RESNET50_LABEL_FILE = "resnet50_labels.txt" if FLAGS.openvino: import openvino.runtime as ov @@ -2701,24 +2008,19 @@ def create_fixed_models( # Make multiple versions of some models for version testing # (they use different version policies when created above) - # BF16 generation: TensorRT requires SM>=8.0; ONNX has no such - # requirement (model generation runs on CPU; ORT decides at runtime). - bf16_for_trt = FLAGS.tensorrt and tu.check_gpus_compute_capability( - min_capability=8.0 - ) - if FLAGS.tensorrt and not bf16_for_trt: - print( - "Skipping the generation of TensorRT PLAN models for the BF16 datatype!" - ) - if bf16_for_trt or FLAGS.onnx: - create_fixed_models( - FLAGS.models_dir, - np_dtype_bfloat16, - np_dtype_bfloat16, - np_dtype_bfloat16, - ) - if FLAGS.tensorrt: + if tu.check_gpus_compute_capability(min_capability=8.0): + create_fixed_models( + FLAGS.models_dir, + np_dtype_bfloat16, + np_dtype_bfloat16, + np_dtype_bfloat16, + ) + else: + print( + "Skipping the generation of TensorRT PLAN models for the BF16 datatype!" + ) + for vt in [np.float32, np.float16, np.int32, np.uint8]: create_plan_modelfile( FLAGS.models_dir, 8, 2, (16,), (16,), (16,), vt, vt, vt, swap=True @@ -2798,7 +2100,6 @@ def create_fixed_models( create_onnx_modelfile( FLAGS.models_dir, 0, 3, (16,), (16,), (16,), vt, vt, vt, swap=True ) - if FLAGS.libtorch: for vt in [np.float32, np.int32, np.int16, np.int8]: create_libtorch_modelfile( @@ -2813,7 +2114,6 @@ def create_fixed_models( create_libtorch_modelfile( FLAGS.models_dir, 0, 3, (16,), (16,), (16,), vt, vt, vt, swap=True ) - if FLAGS.openvino: for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]: create_openvino_modelfile( @@ -2997,26 +2297,22 @@ def create_fixed_models( 32, ) - # BF16 generation: TensorRT requires SM>=8.0; ONNX has no such - # requirement (model generation runs on CPU; ORT decides at runtime). - bf16_for_trt = FLAGS.tensorrt and tu.check_gpus_compute_capability( - min_capability=8.0 - ) - if FLAGS.tensorrt and not bf16_for_trt: - print( - "Skipping the generation of TensorRT PLAN models for the BF16 datatype!" - ) - if bf16_for_trt or FLAGS.onnx: - create_models( - FLAGS.models_dir, - np_dtype_bfloat16, - np_dtype_bfloat16, - np_dtype_bfloat16, - (-1, -1), - (-1, -1), - (-1, -1), - 0, - ) + if FLAGS.tensorrt: + if tu.check_gpus_compute_capability(min_capability=8.0): + create_models( + FLAGS.models_dir, + np_dtype_bfloat16, + np_dtype_bfloat16, + np_dtype_bfloat16, + (-1, -1), + (-1, -1), + (-1, -1), + 0, + ) + else: + print( + "Skipping the generation of TensorRT PLAN models for the BF16 datatype!" + ) if FLAGS.ensemble: # Create utility models used in ensemble @@ -3033,16 +2329,3 @@ def create_fixed_models( # to fixed size model is not safe but doable for model_shape in [(-1,), (-1, -1), (-1, -1, -1)]: emu.create_nop_modelconfig(FLAGS.models_dir, model_shape, model_dtype) - - if FLAGS.torch_aoti: - print( - f"{_color_magenta}PyTorch: Complex AOTI model generation requested{_color_reset}" - ) - if create_torch_aoti_complex_model_file(FLAGS.models_dir): - create_torch_aoti_complex_model_config(FLAGS.models_dir) - - if FLAGS.torchvision_aoti: - # TODO: Add support for variable batch size and version policy for torchvision AOTI models. - print(f"{_color_blue}TorchVision AOTI model generation requested{_color_reset}") - if create_torchvision_aoti_model_file(FLAGS.models_dir, 1): - create_torchvision_aoti_model_config(FLAGS.models_dir, 1) diff --git a/qa/common/gen_qa_ort_scalar_models.py b/qa/common/gen_qa_ort_scalar_models.py index 116a688a39..c00a97d5ed 100755 --- a/qa/common/gen_qa_ort_scalar_models.py +++ b/qa/common/gen_qa_ort_scalar_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -68,7 +68,7 @@ def create_onnx_modelfile(models_dir, shape, dtype, model_version=1): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") @@ -103,7 +103,7 @@ def create_onnx_modelconfig(models_dir, dtype, shape): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: diff --git a/qa/common/gen_qa_ragged_models.py b/qa/common/gen_qa_ragged_models.py index a43888c398..de8c583d88 100755 --- a/qa/common/gen_qa_ragged_models.py +++ b/qa/common/gen_qa_ragged_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -57,11 +57,7 @@ def create_plan_modelfile(models_dir, model_version, dtype): # - BATCH_MAX_ELEMENT_COUNT_AS_SHAPE # - BATCH_ITEM_SHAPE_FLATTEN - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() trt_dtype = np_to_trt_dtype(dtype) @@ -129,7 +125,7 @@ def create_plan_modelfile(models_dir, model_version, dtype): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -261,7 +257,7 @@ def create_onnx_modelfile(models_dir, model_version, dtype): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") @@ -324,13 +320,13 @@ def forward(self, BATCH_INPUT, BATCH_AND_SIZE_INPUT, RAGGED_INPUT): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir traced.save(model_version_dir + "/model.pt") -def create_modelconfig(models_dir, max_batch, dtype, backend, platform): +def create_modelconfig(models_dir, max_batch, model_version, dtype, backend, platform): version_policy_str = "{ latest { num_versions: 1 }}" backend_spec = """ @@ -402,7 +398,7 @@ def create_modelconfig(models_dir, max_batch, dtype, backend, platform): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -416,11 +412,7 @@ def create_plan_itemshape_modelfile(models_dir, model_version, dtype): # generated to have matching batch dimension, the output can be produced # via identity op and expect Triton will scatter the output properly. - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() trt_dtype = np_to_trt_dtype(dtype) @@ -455,7 +447,7 @@ def create_plan_itemshape_modelfile(models_dir, model_version, dtype): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -505,7 +497,7 @@ def create_onnx_itemshape_modelfile(models_dir, model_version, dtype): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") @@ -541,13 +533,15 @@ def forward(self, RAGGED_INPUT, BATCH_INPUT): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir traced.save(model_version_dir + "/model.pt") -def create_itemshape_modelconfig(models_dir, max_batch, dtype, backend, platform): +def create_itemshape_modelconfig( + models_dir, max_batch, model_version, dtype, backend, platform +): version_policy_str = "{ latest { num_versions: 1 }}" backend_spec = """ @@ -599,7 +593,7 @@ def create_itemshape_modelconfig(models_dir, max_batch, dtype, backend, platform try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -609,19 +603,29 @@ def create_itemshape_modelconfig(models_dir, max_batch, dtype, backend, platform def create_batch_input_models(models_dir): model_version = 1 if FLAGS.tensorrt: - create_modelconfig(models_dir, 4, np.float32, "tensorrt", "plan") + create_modelconfig(models_dir, 4, model_version, np.float32, "tensorrt", "plan") create_plan_modelfile(models_dir, model_version, np.float32) - create_itemshape_modelconfig(models_dir, 4, np.float32, "tensorrt", "plan") + create_itemshape_modelconfig( + models_dir, 4, model_version, np.float32, "tensorrt", "plan" + ) create_plan_itemshape_modelfile(models_dir, model_version, np.float32) if FLAGS.onnx: - create_modelconfig(models_dir, 4, np.float32, "onnxruntime", "onnx") + create_modelconfig( + models_dir, 4, model_version, np.float32, "onnxruntime", "onnx" + ) create_onnx_modelfile(models_dir, model_version, np.float32) - create_itemshape_modelconfig(models_dir, 4, np.float32, "onnxruntime", "onnx") + create_itemshape_modelconfig( + models_dir, 4, model_version, np.float32, "onnxruntime", "onnx" + ) create_onnx_itemshape_modelfile(models_dir, model_version, np.float32) if FLAGS.libtorch: - create_modelconfig(models_dir, 4, np.float32, "pytorch", "libtorch") + create_modelconfig( + models_dir, 4, model_version, np.float32, "pytorch", "libtorch" + ) create_libtorch_modelfile(models_dir, model_version, np.float32) - create_itemshape_modelconfig(models_dir, 4, np.float32, "pytorch", "libtorch") + create_itemshape_modelconfig( + models_dir, 4, model_version, np.float32, "pytorch", "libtorch" + ) create_libtorch_itemshape_modelfile(models_dir, model_version, np.float32) diff --git a/qa/common/gen_qa_reshape_models.py b/qa/common/gen_qa_reshape_models.py index 56d75f70ab..8193b29677 100755 --- a/qa/common/gen_qa_reshape_models.py +++ b/qa/common/gen_qa_reshape_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -49,18 +49,16 @@ def create_plan_modelfile( models_dir, model_version, max_batch, dtype, input_shapes, output_shapes ): assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_trt_model(dtype, dtype, dtype): + if not tu.validate_for_trt_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): return trt_dtype = np_to_trt_dtype(dtype) io_cnt = len(input_shapes) # Create the model that copies inputs to corresponding outputs. - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() @@ -116,7 +114,7 @@ def create_plan_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -125,6 +123,7 @@ def create_plan_modelfile( def create_plan_modelconfig( models_dir, + model_version, max_batch, dtype, input_shapes, @@ -135,7 +134,9 @@ def create_plan_modelconfig( assert len(input_shapes) == len(input_model_shapes) assert len(output_shapes) == len(output_model_shapes) assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_trt_model(dtype, dtype, dtype): + if not tu.validate_for_trt_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): return io_cnt = len(input_shapes) @@ -195,7 +196,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -413,7 +414,7 @@ def forward(self, input0, input1, input2, input3): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir traced.save(model_version_dir + "/model.pt") @@ -421,6 +422,7 @@ def forward(self, input0, input1, input2, input3): def create_libtorch_modelconfig( models_dir, + model_version, max_batch, dtype, input_shapes, @@ -500,7 +502,7 @@ def create_libtorch_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -511,7 +513,15 @@ def create_ensemble_modelfile( models_dir, model_version, max_batch, dtype, input_shapes, output_shapes ): assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_ensemble_model("reshape", dtype, dtype, dtype): + if not tu.validate_for_ensemble_model( + "reshape", + dtype, + dtype, + dtype, + input_shapes[0], + input_shapes[0], + input_shapes[0], + ): return emu.create_identity_ensemble_modelfile( @@ -538,7 +548,15 @@ def create_ensemble_modelconfig( assert len(input_shapes) == len(input_model_shapes) assert len(output_shapes) == len(output_model_shapes) assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_ensemble_model("reshape", dtype, dtype, dtype): + if not tu.validate_for_ensemble_model( + "reshape", + dtype, + dtype, + dtype, + input_shapes[0], + input_shapes[0], + input_shapes[0], + ): return # No reason to reshape ensemble inputs / outputs to empty as the inner models @@ -572,7 +590,9 @@ def create_onnx_modelfile( models_dir, model_version, max_batch, dtype, input_shapes, output_shapes ): assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_onnx_model(dtype, dtype, dtype): + if not tu.validate_for_onnx_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): return onnx_dtype = np_to_onnx_dtype(dtype) @@ -632,7 +652,7 @@ def create_onnx_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") @@ -640,6 +660,7 @@ def create_onnx_modelfile( def create_onnx_modelconfig( models_dir, + model_version, max_batch, dtype, input_shapes, @@ -650,7 +671,9 @@ def create_onnx_modelconfig( assert len(input_shapes) == len(input_model_shapes) assert len(output_shapes) == len(output_model_shapes) assert len(input_shapes) == len(output_shapes) - if not tu.validate_for_onnx_model(dtype, dtype, dtype): + if not tu.validate_for_onnx_model( + dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] + ): return io_cnt = len(input_shapes) @@ -677,7 +700,7 @@ def create_onnx_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -700,6 +723,8 @@ def create_openvino_modelfile( dtype, dtype, batch_dim + input_shapes[0], + batch_dim + input_shapes[0], + batch_dim + input_shapes[0], ): return @@ -737,6 +762,7 @@ def create_openvino_modelfile( def create_openvino_modelconfig( models_dir, + model_version, max_batch, dtype, input_shapes, @@ -759,6 +785,8 @@ def create_openvino_modelconfig( dtype, dtype, batch_dim + input_shapes[0], + batch_dim + input_shapes[0], + batch_dim + input_shapes[0], ): return @@ -821,7 +849,7 @@ def create_openvino_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -846,6 +874,7 @@ def create_models( if FLAGS.onnx: create_onnx_modelconfig( models_dir, + model_version, 8, dtype, input_shapes, @@ -859,6 +888,7 @@ def create_models( if no_batch: create_onnx_modelconfig( models_dir, + model_version, 0, dtype, input_shapes, @@ -938,6 +968,7 @@ def create_trt_models( if FLAGS.tensorrt: create_plan_modelconfig( models_dir, + model_version, 8, dtype, input_shapes, @@ -951,6 +982,7 @@ def create_trt_models( if no_batch: create_plan_modelconfig( models_dir, + model_version, 0, dtype, input_shapes, @@ -986,6 +1018,7 @@ def create_libtorch_models( if FLAGS.libtorch: create_libtorch_modelconfig( models_dir, + model_version, 8, dtype, input_shapes, @@ -1000,6 +1033,7 @@ def create_libtorch_models( if no_batch and (dtype != np_dtype_string): create_libtorch_modelconfig( models_dir, + model_version, 0, dtype, input_shapes, @@ -1035,6 +1069,7 @@ def create_openvino_models( if FLAGS.openvino: create_openvino_modelconfig( models_dir, + model_version, 8, dtype, input_shapes, @@ -1048,6 +1083,7 @@ def create_openvino_models( if no_batch: create_openvino_modelconfig( models_dir, + model_version, 0, dtype, input_shapes, diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index f8d89a5f9e..ad31bbc0ba 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -59,11 +59,7 @@ def create_plan_shape_tensor_modelfile( trt_shape_dtype = np_to_trt_dtype(shape_tensor_input_dtype) trt_memory_format = trt.TensorFormat.LINEAR - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() @@ -118,8 +114,7 @@ def create_plan_shape_tensor_modelfile( flags = 1 << int(trt.BuilderFlag.DIRECT_IO) flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) - if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): - flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) + flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) if trt_dtype == trt.int8: flags |= 1 << int(trt.BuilderFlag.INT8) @@ -175,7 +170,7 @@ def create_plan_shape_tensor_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -187,11 +182,7 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): # Create the model. For now don't implement a proper accumulator # just return 0 if not-ready and 'INPUT'+'START' otherwise... the # tests know to expect this. - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() @@ -266,7 +257,7 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -280,11 +271,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) # Create the model. For now don't implement a proper accumulator # just return 0 if not-ready and 'INPUT'+'START' otherwise... the # tests know to expect this. - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() @@ -321,8 +308,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) - if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): - flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) + flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) if trt_dtype == trt.int8: flags |= 1 << int(trt.BuilderFlag.INT8) @@ -384,7 +370,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -392,7 +378,7 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) def create_plan_models(models_dir, model_version, max_batch, dtype, shape): - if not tu.validate_for_trt_model(dtype, dtype, dtype): + if not tu.validate_for_trt_model(dtype, dtype, dtype, shape, shape, shape): return if dtype != np.float32: @@ -402,9 +388,9 @@ def create_plan_models(models_dir, model_version, max_batch, dtype, shape): def create_plan_modelconfig( - models_dir, max_batch, dtype, shape, shape_tensor_input_dtype=None + models_dir, model_version, max_batch, dtype, shape, shape_tensor_input_dtype=None ): - if not tu.validate_for_trt_model(dtype, dtype, dtype): + if not tu.validate_for_trt_model(dtype, dtype, dtype, shape, shape, shape): return model_name = tu.get_sequence_model_name( @@ -561,7 +547,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -569,7 +555,7 @@ def create_plan_modelconfig( def create_onnx_modelfile(models_dir, model_version, max_batch, dtype, shape): - if not tu.validate_for_onnx_model(dtype, dtype, dtype): + if not tu.validate_for_onnx_model(dtype, dtype, dtype, shape, shape, shape): return model_name = tu.get_sequence_model_name( @@ -667,14 +653,14 @@ def create_onnx_modelfile(models_dir, model_version, max_batch, dtype, shape): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir onnx.save(model_def, model_version_dir + "/model.onnx") -def create_onnx_modelconfig(models_dir, max_batch, dtype, shape): - if not tu.validate_for_onnx_model(dtype, dtype, dtype): +def create_onnx_modelconfig(models_dir, model_version, max_batch, dtype, shape): + if not tu.validate_for_onnx_model(dtype, dtype, dtype, shape, shape, shape): return model_name = tu.get_sequence_model_name( @@ -745,7 +731,7 @@ def create_onnx_modelconfig(models_dir, max_batch, dtype, shape): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -798,13 +784,13 @@ def forward(self, input0, start0, ready0): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir traced.save(model_version_dir + "/model.pt") -def create_libtorch_modelconfig(models_dir, max_batch, dtype, shape): +def create_libtorch_modelconfig(models_dir, model_version, max_batch, dtype, shape): if not tu.validate_for_libtorch_model( dtype, dtype, dtype, shape, shape, shape, max_batch ): @@ -882,7 +868,7 @@ def create_libtorch_modelconfig(models_dir, max_batch, dtype, shape): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -897,7 +883,9 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape max_batch, ] ) - if not tu.validate_for_openvino_model(dtype, dtype, dtype, batch_dim + shape): + if not tu.validate_for_openvino_model( + dtype, dtype, dtype, batch_dim + shape, batch_dim + shape, batch_dim + shape + ): return model_name = tu.get_sequence_model_name( @@ -916,7 +904,7 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype, shape openvino_save_model(model_version_dir, model) -def create_openvino_modelconfig(models_dir, max_batch, dtype, shape): +def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype, shape): batch_dim = ( [] if max_batch == 0 @@ -924,7 +912,9 @@ def create_openvino_modelconfig(models_dir, max_batch, dtype, shape): max_batch, ] ) - if not tu.validate_for_openvino_model(dtype, dtype, dtype, batch_dim + shape): + if not tu.validate_for_openvino_model( + dtype, dtype, dtype, batch_dim + shape, batch_dim + shape, batch_dim + shape + ): return model_name = tu.get_sequence_model_name( @@ -984,7 +974,7 @@ def create_openvino_modelconfig(models_dir, max_batch, dtype, shape): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -996,12 +986,16 @@ def create_shape_tensor_models( ): model_version = 1 - create_plan_modelconfig(models_dir, 8, dtype, shape, shape_tensor_input_dtype) + create_plan_modelconfig( + models_dir, model_version, 8, dtype, shape, shape_tensor_input_dtype + ) create_plan_shape_tensor_modelfile( models_dir, model_version, 8, dtype, shape, shape_tensor_input_dtype ) if no_batch: - create_plan_modelconfig(models_dir, 0, dtype, shape, shape_tensor_input_dtype) + create_plan_modelconfig( + models_dir, model_version, 0, dtype, shape, shape_tensor_input_dtype + ) create_plan_shape_tensor_modelfile( models_dir, model_version, 0, dtype, shape, shape_tensor_input_dtype ) @@ -1017,32 +1011,32 @@ def create_models(models_dir, dtype, shape, no_batch=True): if dtype == np.int8: suffix = [1, 1] - create_plan_modelconfig(models_dir, 8, dtype, shape + suffix) + create_plan_modelconfig(models_dir, model_version, 8, dtype, shape + suffix) create_plan_models(models_dir, model_version, 8, dtype, shape + suffix) if no_batch: - create_plan_modelconfig(models_dir, 0, dtype, shape + suffix) + create_plan_modelconfig(models_dir, model_version, 0, dtype, shape + suffix) create_plan_models(models_dir, model_version, 0, dtype, shape + suffix) if FLAGS.onnx: - create_onnx_modelconfig(models_dir, 8, dtype, shape) + create_onnx_modelconfig(models_dir, model_version, 8, dtype, shape) create_onnx_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: - create_onnx_modelconfig(models_dir, 0, dtype, shape) + create_onnx_modelconfig(models_dir, model_version, 0, dtype, shape) create_onnx_modelfile(models_dir, model_version, 0, dtype, shape) # Skip for PyTorch String I/O if FLAGS.libtorch and (dtype != np_dtype_string): - create_libtorch_modelconfig(models_dir, 8, dtype, shape) + create_libtorch_modelconfig(models_dir, model_version, 8, dtype, shape) create_libtorch_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: - create_libtorch_modelconfig(models_dir, 0, dtype, shape) + create_libtorch_modelconfig(models_dir, model_version, 0, dtype, shape) create_libtorch_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.openvino: - create_openvino_modelconfig(models_dir, 8, dtype, shape) + create_openvino_modelconfig(models_dir, model_version, 8, dtype, shape) create_openvino_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: - create_openvino_modelconfig(models_dir, 0, dtype, shape) + create_openvino_modelconfig(models_dir, model_version, 0, dtype, shape) create_openvino_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.ensemble: diff --git a/qa/common/gen_qa_torchtrt_models.py b/qa/common/gen_qa_torchtrt_models.py index 3abfec1e2c..5f6ea04581 100755 --- a/qa/common/gen_qa_torchtrt_models.py +++ b/qa/common/gen_qa_torchtrt_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,21 +28,11 @@ import argparse import os -import sys import torch +import torch_tensorrt import torchvision -try: - import torch_tensorrt -except ImportError: - print( - "WARNING: torch_tensorrt is not available in this environment. " - "Skipping Torch-TensorRT model generation.", - file=sys.stderr, - ) - sys.exit(0) - def create_resnet50_torchtrt(models_dir, max_batch): model = torchvision.models.resnet50(pretrained=True) @@ -71,7 +61,7 @@ def create_resnet50_torchtrt(models_dir, max_batch): try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir torch.jit.save(trt_ts_module, model_version_dir + "/model.pt") @@ -106,7 +96,7 @@ def create_resnet50_torchtrt_modelconfig(models_dir, max_batch): try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: diff --git a/qa/common/gen_qa_trt_data_dependent_shape.py b/qa/common/gen_qa_trt_data_dependent_shape.py index 8984059411..c6f4bf2b5e 100755 --- a/qa/common/gen_qa_trt_data_dependent_shape.py +++ b/qa/common/gen_qa_trt_data_dependent_shape.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,11 +45,7 @@ def create_data_dependent_modelfile( trt_input_dtype = np_to_trt_dtype(input_dtype) # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() @@ -86,17 +82,11 @@ def create_data_dependent_modelfile( # serialized model engine_bytes = builder.build_serialized_network(network, config) - if engine_bytes is None: - print( - f"warning: Skipping {model_name}: TRT engine build failed " - f"(NonZero op may not be supported on this GPU/TRT version)" - ) - return model_version_dir = models_dir + "/" + model_name + "/1" try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -135,7 +125,7 @@ def create_data_dependent_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: diff --git a/qa/common/gen_qa_trt_format_models.py b/qa/common/gen_qa_trt_format_models.py index 5f2cadd69e..6419a6e2ab 100755 --- a/qa/common/gen_qa_trt_format_models.py +++ b/qa/common/gen_qa_trt_format_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -81,11 +81,7 @@ def create_plan_modelfile( trt_output_memory_format = output_memory_format # Create the model - TRT_LOGGER = ( - trt.Logger(trt.Logger.INFO) - if os.environ.get("TRT_VERBOSE") != "1" - else trt.Logger(trt.Logger.VERBOSE) - ) + TRT_LOGGER = trt.Logger(trt.Logger.INFO) builder = trt.Builder(TRT_LOGGER) network = builder.create_network() if max_batch == 0: @@ -106,8 +102,8 @@ def create_plan_modelfile( network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.set_output_type(0, trt_output0_dtype) - out1.set_output_type(0, trt_output1_dtype) + out0.get_output(0).dtype = trt_output0_dtype + out1.get_output(0).dtype = trt_output1_dtype in0.allowed_formats = 1 << int(trt_input_memory_format) in1.allowed_formats = 1 << int(trt_input_memory_format) @@ -147,8 +143,7 @@ def create_plan_modelfile( # The build will fail if TensorRT cannot build an engine without introducing such reformatting. The failure may happen only for some target platforms, because of what formats are supported by kernels for those platforms. # flags = 1 << int(trt.BuilderFlag.DIRECT_IO) flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) - if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): - flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) + flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: if dt == trt.int8: @@ -180,7 +175,7 @@ def create_plan_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -190,6 +185,7 @@ def create_plan_modelfile( def create_plan_modelconfig( models_dir, max_batch, + model_version, input_shape, output0_shape, output1_shape, @@ -204,6 +200,9 @@ def create_plan_modelconfig( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return @@ -325,7 +324,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -349,12 +348,16 @@ def create_plan_model( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): return create_plan_modelconfig( models_dir, max_batch, + model_version, input_shape, output0_shape, output1_shape, diff --git a/qa/common/gen_qa_trt_plugin_models.py b/qa/common/gen_qa_trt_plugin_models.py index 9fd23d92a8..0e2e9cf698 100755 --- a/qa/common/gen_qa_trt_plugin_models.py +++ b/qa/common/gen_qa_trt_plugin_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -44,12 +44,7 @@ def get_trt_plugin(plugin_name): plugin = None field_collection = None - # The upstream onnx_custom_plugin sample is V2 on TRT 10.x release - # branches and V3 on rel-11.0 (and TRT 11 removed the V2 plugin - # registry surface). Pick the matching API at runtime. - registry = trt.get_plugin_registry() - use_v3 = not hasattr(registry, "plugin_creator_list") - plugin_creators = registry.all_creators if use_v3 else registry.plugin_creator_list + plugin_creators = trt.get_plugin_registry().plugin_creator_list for plugin_creator in plugin_creators: if (plugin_creator.name == "CustomHardmax") and ( plugin_name == "CustomHardmax" @@ -62,16 +57,9 @@ def get_trt_plugin(plugin_name): if field_collection is None: raise RuntimeError("Plugin not found: " + plugin_name) - if use_v3: - plugin = plugin_creator.create_plugin( - name=plugin_name, - field_collection=field_collection, - phase=trt.TensorRTPhase.BUILD, - ) - else: - plugin = plugin_creator.create_plugin( - name=plugin_name, field_collection=field_collection - ) + plugin = plugin_creator.create_plugin( + name=plugin_name, field_collection=field_collection + ) return plugin @@ -90,6 +78,9 @@ def create_plan_modelfile( input_dtype, output0_dtype, output0_dtype, + input_shape, + output0_shape, + output0_shape, ): return @@ -116,16 +107,9 @@ def create_plan_modelfile( input_layer = network.add_input( name="INPUT0", dtype=trt_input_dtype, shape=input_with_batchsize ) - # add_plugin_v2 was removed in TRT 11; add_plugin_v3 has existed since - # TRT 10.0. Pick the API that exists on this TRT install; the plugin - # object returned by get_trt_plugin() is matched to the same version. - plugin_obj = get_trt_plugin(plugin_name) - if hasattr(network, "add_plugin_v2"): - plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) - else: - plugin_layer = network.add_plugin_v3( - inputs=[input_layer], shape_inputs=[], plugin=plugin_obj - ) + plugin_layer = network.add_plugin_v2( + inputs=[input_layer], plugin=get_trt_plugin(plugin_name) + ) plugin_layer.get_output(0).name = "OUTPUT0" network.mark_output(plugin_layer.get_output(0)) @@ -163,7 +147,7 @@ def create_plan_modelfile( try: os.makedirs(model_version_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(model_version_dir + "/model.plan", "wb") as f: @@ -173,6 +157,7 @@ def create_plan_modelfile( def create_plan_modelconfig( models_dir, max_batch, + model_version, plugin_name, input_shape, output0_shape, @@ -183,6 +168,9 @@ def create_plan_modelconfig( input_dtype, output0_dtype, output0_dtype, + input_shape, + output0_shape, + output0_shape, ): return @@ -231,7 +219,7 @@ def create_plan_modelconfig( try: os.makedirs(config_dir) - except OSError: + except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: @@ -245,6 +233,7 @@ def create_plugin_models(models_dir): create_plan_modelconfig( models_dir, 8, + model_version, "CustomHardmax", (2, 2), (2, 2), @@ -265,6 +254,7 @@ def create_plugin_models(models_dir): create_plan_modelconfig( models_dir, 0, + model_version, "CustomHardmax", (16, 1, 1), (16, 1, 1), diff --git a/qa/common/infer_test.py b/qa/common/infer_test.py index caa2d4f31e..aa06197373 100755 --- a/qa/common/infer_test.py +++ b/qa/common/infer_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -94,6 +94,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size, 1, 1), + (input_size, 1, 1), + (input_size, 1, 1), ): if "plan" in TEST_BACKENDS: if input_dtype == np.int8: @@ -117,6 +120,9 @@ def _infer_exact_helper( input_dtype, output0_dtype, output1_dtype, + (input_size,), + (input_size,), + (input_size,), ): if "onnx" in TEST_BACKENDS: _infer_exact_helper( diff --git a/qa/common/infer_util.py b/qa/common/infer_util.py index 7e9c775570..edaf0ede47 100755 --- a/qa/common/infer_util.py +++ b/qa/common/infer_util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -1367,13 +1367,11 @@ def shm_basic_infer( big_shm_name="", big_shm_size=64, default_shm_byte_size=64, - register_offset=0, shm_output_offset=0, shm_output_byte_size=64, protocol="http", use_system_shared_memory=False, use_cuda_shared_memory=False, - override_model_name=None, ): # Lazy shm imports... if use_system_shared_memory: @@ -1383,16 +1381,6 @@ def shm_basic_infer( else: raise Exception("No shared memory type specified") - if override_model_name is None: - model_name = "simple" - else: - model_name = override_model_name - - if model_name.startswith("libtorch"): - output_names = ["OUTPUT__0", "OUTPUT__1"] - else: - output_names = ["OUTPUT0", "OUTPUT1"] - input0_data = np.arange(start=0, stop=16, dtype=np.int32) input1_data = np.ones(shape=16, dtype=np.int32) inputs = [] @@ -1400,17 +1388,13 @@ def shm_basic_infer( if protocol == "http": inputs.append(httpclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(httpclient.InferInput("INPUT1", [1, 16], "INT32")) - outputs.append( - httpclient.InferRequestedOutput(output_names[0], binary_data=True) - ) - outputs.append( - httpclient.InferRequestedOutput(output_names[1], binary_data=False) - ) + outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) + outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=False)) else: inputs.append(grpcclient.InferInput("INPUT0", [1, 16], "INT32")) inputs.append(grpcclient.InferInput("INPUT1", [1, 16], "INT32")) - outputs.append(grpcclient.InferRequestedOutput(output_names[0])) - outputs.append(grpcclient.InferRequestedOutput(output_names[1])) + outputs.append(grpcclient.InferRequestedOutput("OUTPUT0")) + outputs.append(grpcclient.InferRequestedOutput("OUTPUT1")) inputs[0].set_shared_memory("input0_data", default_shm_byte_size) @@ -1430,9 +1414,9 @@ def shm_basic_infer( try: results = triton_client.infer( - model_name, inputs, model_version="", outputs=outputs + "simple", inputs, model_version="", outputs=outputs ) - output = results.get_output(output_names[0]) + output = results.get_output("OUTPUT0") if protocol == "http": output_datatype = output["datatype"] output_shape = output["shape"] @@ -1443,16 +1427,11 @@ def shm_basic_infer( if use_system_shared_memory: output_data = shm.get_contents_as_numpy( - shm_op0_handle, - output_dtype, - output_shape, - offset=register_offset + shm_output_offset, + shm_op0_handle, output_dtype, output_shape ) elif use_cuda_shared_memory: output_data = cudashm.get_contents_as_numpy( - shm_op0_handle, - output_dtype, - output_shape, + shm_op0_handle, output_dtype, output_shape ) tester.assertTrue( diff --git a/qa/common/resnet50_labels.txt b/qa/common/resnet50_labels.txt deleted file mode 100644 index f40829ed0f..0000000000 --- a/qa/common/resnet50_labels.txt +++ /dev/null @@ -1,1000 +0,0 @@ -tench -goldfish -great white shark -tiger shark -hammerhead -electric ray -stingray -cock -hen -ostrich -brambling -goldfinch -house finch -junco -indigo bunting -robin -bulbul -jay -magpie -chickadee -water ouzel -kite -bald eagle -vulture -great grey owl -European fire salamander -common newt -eft -spotted salamander -axolotl -bullfrog -tree frog -tailed frog -loggerhead -leatherback turtle -mud turtle -terrapin -box turtle -banded gecko -common iguana -American chameleon -whiptail -agama -frilled lizard -alligator lizard -Gila monster -green lizard -African chameleon -Komodo dragon -African crocodile -American alligator -triceratops -thunder snake -ringneck snake -hognose snake -green snake -king snake -garter snake -water snake -vine snake -night snake -boa constrictor -rock python -Indian cobra -green mamba -sea snake -horned viper -diamondback -sidewinder -trilobite -harvestman -scorpion -black and gold garden spider -barn spider -garden spider -black widow -tarantula -wolf spider -tick -centipede -black grouse -ptarmigan -ruffed grouse -prairie chicken -peacock -quail -partridge -African grey -macaw -sulphur-crested cockatoo -lorikeet -coucal -bee eater -hornbill -hummingbird -jacamar -toucan -drake -red-breasted merganser -goose -black swan -tusker -echidna -platypus -wallaby -koala -wombat -jellyfish -sea anemone -brain coral -flatworm -nematode -conch -snail -slug -sea slug -chiton -chambered nautilus -Dungeness crab -rock crab -fiddler crab -king crab -American lobster -spiny lobster -crayfish -hermit crab -isopod -white stork -black stork -spoonbill -flamingo -little blue heron -American egret -bittern -crane -limpkin -European gallinule -American coot -bustard -ruddy turnstone -red-backed sandpiper -redshank -dowitcher -oystercatcher -pelican -king penguin -albatross -grey whale -killer whale -dugong -sea lion -Chihuahua -Japanese spaniel -Maltese dog -Pekinese -Shih-Tzu -Blenheim spaniel -papillon -toy terrier -Rhodesian ridgeback -Afghan hound -basset -beagle -bloodhound -bluetick -black-and-tan coonhound -Walker hound -English foxhound -redbone -borzoi -Irish wolfhound -Italian greyhound -whippet -Ibizan hound -Norwegian elkhound -otterhound -Saluki -Scottish deerhound -Weimaraner -Staffordshire bullterrier -American Staffordshire terrier -Bedlington terrier -Border terrier -Kerry blue terrier -Irish terrier -Norfolk terrier -Norwich terrier -Yorkshire terrier -wire-haired fox terrier -Lakeland terrier -Sealyham terrier -Airedale -cairn -Australian terrier -Dandie Dinmont -Boston bull -miniature schnauzer -giant schnauzer -standard schnauzer -Scotch terrier -Tibetan terrier -silky terrier -soft-coated wheaten terrier -West Highland white terrier -Lhasa -flat-coated retriever -curly-coated retriever -golden retriever -Labrador retriever -Chesapeake Bay retriever -German short-haired pointer -vizsla -English setter -Irish setter -Gordon setter -Brittany spaniel -clumber -English springer -Welsh springer spaniel -cocker spaniel -Sussex spaniel -Irish water spaniel -kuvasz -schipperke -groenendael -malinois -briard -kelpie -komondor -Old English sheepdog -Shetland sheepdog -collie -Border collie -Bouvier des Flandres -Rottweiler -German shepherd -Doberman -miniature pinscher -Greater Swiss Mountain dog -Bernese mountain dog -Appenzeller -EntleBucher -boxer -bull mastiff -Tibetan mastiff -French bulldog -Great Dane -Saint Bernard -Eskimo dog -malamute -Siberian husky -dalmatian -affenpinscher -basenji -pug -Leonberg -Newfoundland -Great Pyrenees -Samoyed -Pomeranian -chow -keeshond -Brabancon griffon -Pembroke -Cardigan -toy poodle -miniature poodle -standard poodle -Mexican hairless -timber wolf -white wolf -red wolf -coyote -dingo -dhole -African hunting dog -hyena -red fox -kit fox -Arctic fox -grey fox -tabby -tiger cat -Persian cat -Siamese cat -Egyptian cat -cougar -lynx -leopard -snow leopard -jaguar -lion -tiger -cheetah -brown bear -American black bear -ice bear -sloth bear -mongoose -meerkat -tiger beetle -ladybug -ground beetle -long-horned beetle -leaf beetle -dung beetle -rhinoceros beetle -weevil -fly -bee -ant -grasshopper -cricket -walking stick -cockroach -mantis -cicada -leafhopper -lacewing -dragonfly -damselfly -admiral -ringlet -monarch -cabbage butterfly -sulphur butterfly -lycaenid -starfish -sea urchin -sea cucumber -wood rabbit -hare -Angora -hamster -porcupine -fox squirrel -marmot -beaver -guinea pig -sorrel -zebra -hog -wild boar -warthog -hippopotamus -ox -water buffalo -bison -ram -bighorn -ibex -hartebeest -impala -gazelle -Arabian camel -llama -weasel -mink -polecat -black-footed ferret -otter -skunk -badger -armadillo -three-toed sloth -orangutan -gorilla -chimpanzee -gibbon -siamang -guenon -patas -baboon -macaque -langur -colobus -proboscis monkey -marmoset -capuchin -howler monkey -titi -spider monkey -squirrel monkey -Madagascar cat -indri -Indian elephant -African elephant -lesser panda -giant panda -barracouta -eel -coho -rock beauty -anemone fish -sturgeon -gar -lionfish -puffer -abacus -abaya -academic gown -accordion -acoustic guitar -aircraft carrier -airliner -airship -altar -ambulance -amphibian -analog clock -apiary -apron -ashcan -assault rifle -backpack -bakery -balance beam -balloon -ballpoint -Band Aid -banjo -bannister -barbell -barber chair -barbershop -barn -barometer -barrel -barrow -baseball -basketball -bassinet -bassoon -bathing cap -bath towel -bathtub -beach wagon -beacon -beaker -bearskin -beer bottle -beer glass -bell cote -bib -bicycle-built-for-two -bikini -binder -binoculars -birdhouse -boathouse -bobsled -bolo tie -bonnet -bookcase -bookshop -bottlecap -bow -bow tie -brass -brassiere -breakwater -breastplate -broom -bucket -buckle -bulletproof vest -bullet train -butcher shop -cab -caldron -candle -cannon -canoe -can opener -cardigan -car mirror -carousel -carpenter's kit -carton -car wheel -cash machine -cassette -cassette player -castle -catamaran -CD player -cello -cellular telephone -chain -chainlink fence -chain mail -chain saw -chest -chiffonier -chime -china cabinet -Christmas stocking -church -cinema -cleaver -cliff dwelling -cloak -clog -cocktail shaker -coffee mug -coffeepot -coil -combination lock -computer keyboard -confectionery -container ship -convertible -corkscrew -cornet -cowboy boot -cowboy hat -cradle -crane -crash helmet -crate -crib -Crock Pot -croquet ball -crutch -cuirass -dam -desk -desktop computer -dial telephone -diaper -digital clock -digital watch -dining table -dishrag -dishwasher -disk brake -dock -dogsled -dome -doormat -drilling platform -drum -drumstick -dumbbell -Dutch oven -electric fan -electric guitar -electric locomotive -entertainment center -envelope -espresso maker -face powder -feather boa -file -fireboat -fire engine -fire screen -flagpole -flute -folding chair -football helmet -forklift -fountain -fountain pen -four-poster -freight car -French horn -frying pan -fur coat -garbage truck -gasmask -gas pump -goblet -go-kart -golf ball -golfcart -gondola -gong -gown -grand piano -greenhouse -grille -grocery store -guillotine -hair slide -hair spray -half track -hammer -hamper -hand blower -hand-held computer -handkerchief -hard disc -harmonica -harp -harvester -hatchet -holster -home theater -honeycomb -hook -hoopskirt -horizontal bar -horse cart -hourglass -iPod -iron -jack-o'-lantern -jean -jeep -jersey -jigsaw puzzle -jinrikisha -joystick -kimono -knee pad -knot -lab coat -ladle -lampshade -laptop -lawn mower -lens cap -letter opener -library -lifeboat -lighter -limousine -liner -lipstick -Loafer -lotion -loudspeaker -loupe -lumbermill -magnetic compass -mailbag -mailbox -maillot -maillot -manhole cover -maraca -marimba -mask -matchstick -maypole -maze -measuring cup -medicine chest -megalith -microphone -microwave -military uniform -milk can -minibus -miniskirt -minivan -missile -mitten -mixing bowl -mobile home -Model T -modem -monastery -monitor -moped -mortar -mortarboard -mosque -mosquito net -motor scooter -mountain bike -mountain tent -mouse -mousetrap -moving van -muzzle -nail -neck brace -necklace -nipple -notebook -obelisk -oboe -ocarina -odometer -oil filter -organ -oscilloscope -overskirt -oxcart -oxygen mask -packet -paddle -paddlewheel -padlock -paintbrush -pajama -palace -panpipe -paper towel -parachute -parallel bars -park bench -parking meter -passenger car -patio -pay-phone -pedestal -pencil box -pencil sharpener -perfume -Petri dish -photocopier -pick -pickelhaube -picket fence -pickup -pier -piggy bank -pill bottle -pillow -ping-pong ball -pinwheel -pirate -pitcher -plane -planetarium -plastic bag -plate rack -plow -plunger -Polaroid camera -pole -police van -poncho -pool table -pop bottle -pot -potter's wheel -power drill -prayer rug -printer -prison -projectile -projector -puck -punching bag -purse -quill -quilt -racer -racket -radiator -radio -radio telescope -rain barrel -recreational vehicle -reel -reflex camera -refrigerator -remote control -restaurant -revolver -rifle -rocking chair -rotisserie -rubber eraser -rugby ball -rule -running shoe -safe -safety pin -saltshaker -sandal -sarong -sax -scabbard -scale -school bus -schooner -scoreboard -screen -screw -screwdriver -seat belt -sewing machine -shield -shoe shop -shoji -shopping basket -shopping cart -shovel -shower cap -shower curtain -ski -ski mask -sleeping bag -slide rule -sliding door -slot -snorkel -snowmobile -snowplow -soap dispenser -soccer ball -sock -solar dish -sombrero -soup bowl -space bar -space heater -space shuttle -spatula -speedboat -spider web -spindle -sports car -spotlight -stage -steam locomotive -steel arch bridge -steel drum -stethoscope -stole -stone wall -stopwatch -stove -strainer -streetcar -stretcher -studio couch -stupa -submarine -suit -sundial -sunglass -sunglasses -sunscreen -suspension bridge -swab -sweatshirt -swimming trunks -swing -switch -syringe -table lamp -tank -tape player -teapot -teddy -television -tennis ball -thatch -theater curtain -thimble -thresher -throne -tile roof -toaster -tobacco shop -toilet seat -torch -totem pole -tow truck -toyshop -tractor -trailer truck -tray -trench coat -tricycle -trimaran -tripod -triumphal arch -trolleybus -trombone -tub -turnstile -typewriter keyboard -umbrella -unicycle -upright -vacuum -vase -vault -velvet -vending machine -vestment -viaduct -violin -volleyball -waffle iron -wall clock -wallet -wardrobe -warplane -washbasin -washer -water bottle -water jug -water tower -whiskey jug -whistle -wig -window screen -window shade -Windsor tie -wine bottle -wing -wok -wooden spoon -wool -worm fence -wreck -yawl -yurt -web site -comic book -crossword puzzle -street sign -traffic light -book jacket -menu -plate -guacamole -consomme -hot pot -trifle -ice cream -ice lolly -French loaf -bagel -pretzel -cheeseburger -hotdog -mashed potato -head cabbage -broccoli -cauliflower -zucchini -spaghetti squash -acorn squash -butternut squash -cucumber -artichoke -bell pepper -cardoon -mushroom -Granny Smith -strawberry -orange -lemon -fig -pineapple -banana -jackfruit -custard apple -pomegranate -hay -carbonara -chocolate sauce -dough -meat loaf -pizza -potpie -burrito -red wine -espresso -cup -eggnog -alp -bubble -cliff -coral reef -geyser -lakeside -promontory -sandbar -seashore -valley -volcano -ballplayer -groom -scuba diver -rapeseed -daisy -yellow lady's slipper -corn -acorn -hip -buckeye -coral fungus -agaric -gyromitra -stinkhorn -earthstar -hen-of-the-woods -bolete -ear -toilet tissue diff --git a/qa/common/shm_util.py b/qa/common/shm_util.py index 6f66b422b7..0e533bcdbb 100755 --- a/qa/common/shm_util.py +++ b/qa/common/shm_util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,6 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import threading import time from ctypes import * from os import listdir @@ -36,8 +35,6 @@ import tritonclient.http as httpclient from tritonclient.utils import * -CREATION_LOCK = threading.Lock() - # By default, find tritonserver on "localhost", but can be overridden # with TRITONSERVER_IPADDR envvar _tritonserver_ipaddr = os.environ.get("TRITONSERVER_IPADDR", "localhost") @@ -97,69 +94,64 @@ def create_set_shm_regions( shm_op0_handle = None shm_op1_handle = None - with CREATION_LOCK: - if use_system_shared_memory: - shm_ip0_handle = shm.create_shared_memory_region( - shm_region_names[0] + "_data", - "/" + shm_region_names[0], - input0_byte_size, - ) - shm_ip1_handle = shm.create_shared_memory_region( - shm_region_names[1] + "_data", - "/" + shm_region_names[1], - input1_byte_size, - ) + if use_system_shared_memory: + shm_ip0_handle = shm.create_shared_memory_region( + shm_region_names[0] + "_data", "/" + shm_region_names[0], input0_byte_size + ) + shm_ip1_handle = shm.create_shared_memory_region( + shm_region_names[1] + "_data", "/" + shm_region_names[1], input1_byte_size + ) - i = 0 - if "OUTPUT0" in outputs: - if precreated_shm_regions is None: - shm_op0_handle = shm.create_shared_memory_region( - shm_region_names[2] + "_data", - "/" + shm_region_names[2], - output0_byte_size, - ) - else: - shm_op0_handle = precreated_shm_regions[0] - i += 1 - if "OUTPUT1" in outputs: - if precreated_shm_regions is None: - shm_op1_handle = shm.create_shared_memory_region( - shm_region_names[2 + i] + "_data", - "/" + shm_region_names[2 + i], - output1_byte_size, - ) - else: - shm_op1_handle = precreated_shm_regions[i] + i = 0 + if "OUTPUT0" in outputs: + if precreated_shm_regions is None: + shm_op0_handle = shm.create_shared_memory_region( + shm_region_names[2] + "_data", + "/" + shm_region_names[2], + output0_byte_size, + ) + else: + shm_op0_handle = precreated_shm_regions[0] + i += 1 + if "OUTPUT1" in outputs: + if precreated_shm_regions is None: + shm_op1_handle = shm.create_shared_memory_region( + shm_region_names[2 + i] + "_data", + "/" + shm_region_names[2 + i], + output1_byte_size, + ) + else: + shm_op1_handle = precreated_shm_regions[i] - shm.set_shared_memory_region(shm_ip0_handle, input0_list) - shm.set_shared_memory_region(shm_ip1_handle, input1_list) + shm.set_shared_memory_region(shm_ip0_handle, input0_list) + shm.set_shared_memory_region(shm_ip1_handle, input1_list) - if use_cuda_shared_memory: - shm_ip0_handle = cudashm.create_shared_memory_region( - shm_region_names[0] + "_data", input0_byte_size, 0 - ) - shm_ip1_handle = cudashm.create_shared_memory_region( - shm_region_names[1] + "_data", input1_byte_size, 0 - ) - i = 0 - if "OUTPUT0" in outputs: - if precreated_shm_regions is None: - shm_op0_handle = cudashm.create_shared_memory_region( - shm_region_names[2] + "_data", output0_byte_size, 0 - ) - else: - shm_op0_handle = precreated_shm_regions[0] - i += 1 - if "OUTPUT1" in outputs: - if precreated_shm_regions is None: - shm_op1_handle = cudashm.create_shared_memory_region( - shm_region_names[2 + i] + "_data", output1_byte_size, 0 - ) - else: - shm_op1_handle = precreated_shm_regions[i] + if use_cuda_shared_memory: + shm_ip0_handle = cudashm.create_shared_memory_region( + shm_region_names[0] + "_data", input0_byte_size, 0 + ) + shm_ip1_handle = cudashm.create_shared_memory_region( + shm_region_names[1] + "_data", input1_byte_size, 0 + ) + i = 0 + if "OUTPUT0" in outputs: + if precreated_shm_regions is None: + shm_op0_handle = cudashm.create_shared_memory_region( + shm_region_names[2] + "_data", output0_byte_size, 0 + ) + else: + shm_op0_handle = precreated_shm_regions[0] + i += 1 + if "OUTPUT1" in outputs: + if precreated_shm_regions is None: + shm_op1_handle = cudashm.create_shared_memory_region( + shm_region_names[2 + i] + "_data", output1_byte_size, 0 + ) + else: + shm_op1_handle = precreated_shm_regions[i] - cudashm.set_shared_memory_region(shm_ip0_handle, input0_list) - cudashm.set_shared_memory_region(shm_ip1_handle, input1_list) + cudashm.set_shared_memory_region(shm_ip0_handle, input0_list) + cudashm.set_shared_memory_region(shm_ip1_handle, input1_list) return shm_region_names, [ shm_ip0_handle, @@ -345,27 +337,22 @@ def create_set_either_shm_region( if not (use_system_shared_memory or use_cuda_shared_memory): return [] - with CREATION_LOCK: - if use_cuda_shared_memory: - shm_ip_handle = cudashm.create_shared_memory_region( - shm_region_names[0] + "_data", input_byte_size, 0 - ) - shm_op_handle = cudashm.create_shared_memory_region( - shm_region_names[1] + "_data", output_byte_size, 0 - ) - cudashm.set_shared_memory_region(shm_ip_handle, input_list) - elif use_system_shared_memory: - shm_ip_handle = shm.create_shared_memory_region( - shm_region_names[0] + "_data", - "/" + shm_region_names[0], - input_byte_size, - ) - shm_op_handle = shm.create_shared_memory_region( - shm_region_names[1] + "_data", - "/" + shm_region_names[1], - output_byte_size, - ) - shm.set_shared_memory_region(shm_ip_handle, input_list) + if use_cuda_shared_memory: + shm_ip_handle = cudashm.create_shared_memory_region( + shm_region_names[0] + "_data", input_byte_size, 0 + ) + shm_op_handle = cudashm.create_shared_memory_region( + shm_region_names[1] + "_data", output_byte_size, 0 + ) + cudashm.set_shared_memory_region(shm_ip_handle, input_list) + elif use_system_shared_memory: + shm_ip_handle = shm.create_shared_memory_region( + shm_region_names[0] + "_data", "/" + shm_region_names[0], input_byte_size + ) + shm_op_handle = shm.create_shared_memory_region( + shm_region_names[1] + "_data", "/" + shm_region_names[1], output_byte_size + ) + shm.set_shared_memory_region(shm_ip_handle, input_list) return [shm_ip_handle, shm_op_handle] diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 46a42668bf..f3bbcdb16f 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,56 +27,15 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json -import os import unittest -import ml_dtypes import numpy as np _last_request_id = 0 # Numpy does not support the BF16 datatype natively. -# We use ml_dtypes.bfloat16 for BF16. -np_dtype_bfloat16 = ml_dtypes.bfloat16 - -MIB = 2**20 # 1 MIB = 1,048,576 bytes -GIB = 2**30 # 1 GIB = 1,073,741,824 bytes - - -def get_server_process_from_env(env_var="SERVER_PID"): - """ - Return a psutil.Process for the tritonserver under test. - """ - import psutil - - pid_str = os.environ.get(env_var) - if not pid_str: - raise AssertionError(f"{env_var} env var is not set") - try: - return psutil.Process(int(pid_str)) - except (ValueError, psutil.NoSuchProcess) as e: - raise AssertionError(f"Invalid or stale {env_var}={pid_str!r}: {e}") - - -def wait_for_stable_rss(server, rss_tolerance_bytes=0.1 * MIB, stable_threshold=10): - """ - Wait until the RSS of the server is stable. - """ - import time - - last_rss = None - stable_count = 0 - while True: - rss = server.memory_info().rss - if last_rss is not None and abs(rss - last_rss) < rss_tolerance_bytes: - stable_count += 1 - else: - stable_count = 0 - last_rss = rss - if stable_count >= stable_threshold: - break - time.sleep(0.1) - return +# We use this dummy dtype as a representative for BF16. +np_dtype_bfloat16 = np.dtype([("bf16", object)]) def shape_element_count(shape): @@ -116,13 +75,7 @@ def shape_to_dims_str(shape): def validate_for_trt_model( - input_dtype, - output0_dtype, - output1_dtype, - # Unused arguments for consistency with validate_for_libtorch_model - input_shape=None, - output0_shape=None, - output1_shape=None, + input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape ): """Return True if input and output dtypes are supported by a TRT model.""" supported_datatypes = [ @@ -160,6 +113,9 @@ def validate_for_ensemble_model( input_dtype, output0_dtype, output1_dtype, + input_shape, + output0_shape, + output1_shape, ): """Return True if input and output dtypes are supported by the ensemble type.""" @@ -192,13 +148,7 @@ def validate_for_ensemble_model( def validate_for_onnx_model( - input_dtype, - output0_dtype, - output1_dtype, - # Unused arguments for consistency with validate_for_libtorch_model - input_shape=None, - output0_shape=None, - output1_shape=None, + input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape ): """Return True if input and output dtypes are supported by a Onnx model.""" @@ -277,10 +227,7 @@ def validate_for_libtorch_model( def validate_for_openvino_model( - input_dtype, - output0_dtype, - output1_dtype, - input_shape, + input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape ): """Return True if input and output dtypes are supported by an OpenVino model.""" @@ -319,15 +266,12 @@ def get_dtype_name(dtype): def get_model_name(pf, input_dtype, output0_dtype, output1_dtype): - if output1_dtype is None: - return f"{pf}_{get_dtype_name(input_dtype)}_{get_dtype_name(output0_dtype)}" - else: - return "{}_{}_{}_{}".format( - pf, - get_dtype_name(input_dtype), - get_dtype_name(output0_dtype), - get_dtype_name(output1_dtype), - ) + return "{}_{}_{}_{}".format( + pf, + get_dtype_name(input_dtype), + get_dtype_name(output0_dtype), + get_dtype_name(output1_dtype), + ) def get_sequence_model_name(pf, dtype): @@ -365,37 +309,17 @@ def check_gpus_compute_capability(min_capability): Returns: bool """ + import pycuda.driver as cuda - import importlib.util - - if importlib.util.find_spec("cuda") is not None: - from cuda.core import Device - - devices = Device.get_all_devices() - for device in devices: - cc = device.compute_capability - compute_capability_value = cc.major + cc.minor / 10.0 - if compute_capability_value < min_capability: - return False - - elif importlib.util.find_spec("pycuda") is not None: - import pycuda.driver as cuda - - cuda.init() + cuda.init() - for device_index in range(cuda.Device.count()): - device = cuda.Device(device_index) - compute_capability = device.compute_capability() - compute_capability_value = ( - compute_capability[0] + compute_capability[1] / 10.0 - ) + for device_index in range(cuda.Device.count()): + device = cuda.Device(device_index) + compute_capability = device.compute_capability() + compute_capability_value = compute_capability[0] + compute_capability[1] / 10.0 - if compute_capability_value < min_capability: - return False - else: - raise RuntimeError( - "No packages found to determine the compute capability. Please check the environment." - ) + if compute_capability_value < min_capability: + return False return True diff --git a/qa/common/trtllm_util.sh b/qa/common/trtllm_util.sh deleted file mode 100755 index 81ecb2d770..0000000000 --- a/qa/common/trtllm_util.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -function clone_tensorrt_llm_backend_repo { - rm -rf $TENSORRTLLM_BACKEND_DIR && mkdir $TENSORRTLLM_BACKEND_DIR - apt-get update && apt-get install git-lfs -y --no-install-recommends - git clone --single-branch --depth=1 -b ${TENSORRTLLM_BACKEND_REPO_TAG} ${TRITON_REPO_ORG}/tensorrtllm_backend.git $TENSORRTLLM_BACKEND_DIR - cd $TENSORRTLLM_BACKEND_DIR && git lfs install && git submodule update --init --recursive -} - -function build_gpt2_base_model { - # Download weights from HuggingFace Transformers - cd ${GPT_DIR} && rm -rf gpt2 && git clone https://huggingface.co/gpt2-medium gpt2 && cd gpt2 - rm pytorch_model.bin model.safetensors - if ! wget -q https://huggingface.co/gpt2-medium/resolve/main/pytorch_model.bin; then - echo "Downloading pytorch_model.bin failed." - exit 1 - fi - cd ${GPT_DIR} - - # Convert weights from HF Tranformers to FT format - python3 convert_checkpoint.py --model_dir gpt2 --dtype float16 --tp_size ${NUM_GPUS} --output_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" - cd ${BASE_DIR} -} - -function build_gpt2_tensorrt_engine { - # Build TensorRT engines - cd ${GPT_DIR} - trtllm-build --checkpoint_dir "./c-model/gpt2/${NUM_GPUS}-gpu/" \ - --gpt_attention_plugin float16 \ - --remove_input_padding enable \ - --paged_kv_cache enable \ - --gemm_plugin float16 \ - --workers "${NUM_GPUS}" \ - --output_dir "${ENGINES_DIR}" - - cd ${BASE_DIR} -} - -function replace_config_tags { - tag_to_replace="${1}" - new_value="${2}" - config_file_path="${3}" - sed -i "s|${tag_to_replace}|${new_value}|g" ${config_file_path} -} - -function prepare_model_repository { - rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} - cp -r ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} - rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls - mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" - - replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" - - replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" - replace_config_tags '${prompt_embedding_table_data_type}' 'TYPE_FP16' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" -} - -# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on -# success, 1 on failure -function wait_for_server_ready() { - local wait_time_secs="${1:-30}" - shift - local spids=("$@") - - WAIT_RET=0 - - for _ in $(seq "$wait_time_secs"); do - for pid in "${spids[@]}"; do - if ! kill -0 "$pid" >/dev/null 2>&1; then - echo "=== Server not running." - WAIT_RET=1 - return - fi - done - - sleep 1 - - if curl -s --fail localhost:8000/v2/health/ready && - curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then - return - fi - done - - echo "=== Timeout $wait_time_secs secs. Server not ready." - WAIT_RET=1 -} - -function run_server { - python3 ${TENSORRTLLM_BACKEND_DIR}/tensorrt_llm/triton_backend/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & - sleep 2 # allow time to obtain the pid(s) - # Read PIDs into an array, trimming whitespaces - readarray -t SERVER_PID < <(pgrep "tritonserver") - - wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" - if [ "$WAIT_RET" != "0" ]; then - # Cleanup - kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi -} - -function kill_server { - pgrep tritonserver | xargs kill -SIGINT - for pid in "${SERVER_PID[@]}"; do - echo "Waiting for proc ${pid} to terminate..." - while kill -0 $pid >/dev/null 2>&1; do - sleep 1 - done - done -} diff --git a/qa/common/util.sh b/qa/common/util.sh index 08d10d7ed0..3874916573 100755 --- a/qa/common/util.sh +++ b/qa/common/util.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -196,12 +196,7 @@ function run_server () { echo "=== Running LD_PRELOAD=$SERVER_LD_PRELOAD $SERVER $SERVER_ARGS" fi - # If SERVER_ERROR_LOG is not set, redirect stderr to stdout - if [ -z "${SERVER_ERROR_LOG:-}" ]; then - LD_PRELOAD=$SERVER_LD_PRELOAD:${LD_PRELOAD} $SERVER $SERVER_ARGS > $SERVER_LOG 2>&1 & - else - LD_PRELOAD=$SERVER_LD_PRELOAD:${LD_PRELOAD} $SERVER $SERVER_ARGS > $SERVER_LOG 2>$SERVER_ERROR_LOG & - fi + LD_PRELOAD=$SERVER_LD_PRELOAD:${LD_PRELOAD} $SERVER $SERVER_ARGS > $SERVER_LOG 2>&1 & SERVER_PID=$! wait_for_server_ready $SERVER_PID $SERVER_TIMEOUT diff --git a/qa/python_models/bls_memory/model.py b/qa/python_models/bls_memory/model.py index a55da15774..69da4f440f 100644 --- a/qa/python_models/bls_memory/model.py +++ b/qa/python_models/bls_memory/model.py @@ -1,4 +1,4 @@ -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -60,7 +60,7 @@ def test_bls_out_of_memory(self): input0_np, infer_response = self._send_identity_tensor( tensor_size, self._is_decoupled ) - out_of_memory_message = "Failed to increase the shared memory pool size" + out_of_memory_message = "Failed to increase the shared memory pool size for key" if infer_response.has_error(): self.assertIn(out_of_memory_message, infer_response.error().message()) diff --git a/qa/python_models/bls_memory_async/model.py b/qa/python_models/bls_memory_async/model.py index c96073eac0..d9e676b42e 100644 --- a/qa/python_models/bls_memory_async/model.py +++ b/qa/python_models/bls_memory_async/model.py @@ -1,4 +1,4 @@ -# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,7 +53,7 @@ async def test_bls_out_of_memory(): tensor_size = 256 * 1024 * 1024 input0_np, infer_response = await _send_identity_tensor(tensor_size, is_decoupled) - out_of_memory_message = "Failed to increase the shared memory pool size" + out_of_memory_message = "Failed to increase the shared memory pool size for key" if infer_response.has_error(): if not (out_of_memory_message in infer_response.error().message()): diff --git a/qa/python_models/decoupled_grpc_error/config.pbtxt b/qa/python_models/decoupled_grpc_error/config.pbtxt deleted file mode 100644 index 2a532b1f32..0000000000 --- a/qa/python_models/decoupled_grpc_error/config.pbtxt +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -backend: "python" -max_batch_size: 0 - -model_transaction_policy { - decoupled: true -} - -input [ - { - name: "IN" - data_type: TYPE_FP32 - dims: [ -1 ] - }, - { - name: "MODE" - data_type: TYPE_STRING - dims: [ 1 ] - } -] - -output [ - { - name: "OUT" - data_type: TYPE_FP32 - dims: [ -1 ] - } -] - -instance_group [ - { - count: 1 - kind : KIND_CPU - } -] diff --git a/qa/python_models/decoupled_grpc_error/model.py b/qa/python_models/decoupled_grpc_error/model.py deleted file mode 100644 index b498763923..0000000000 --- a/qa/python_models/decoupled_grpc_error/model.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json -import threading -import time - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Decoupled model that always sends an error. - MODE input: - "MODE_ERROR_ONLY" (error response without FINAL flag), - "MODE_ERROR_FINAL" (error response with FINAL flag), - "MODE_ERROR_WITH_DELAYED_FINAL" (error response then delayed FINAL response). - Used to test gRPC server error handling behavior (e.g. triton_grpc_error stream closure and subsequent responses handling). - """ - - def initialize(self, args): - self.model_config = json.loads(args["model_config"]) - if not pb_utils.using_decoupled_model_transaction_policy(self.model_config): - raise pb_utils.TritonModelException( - "This model requires decoupled transaction policy" - ) - self.inflight_thread_count = 0 - self.inflight_thread_count_lck = threading.Lock() - - def execute(self, requests): - for request in requests: - in_tensor = pb_utils.get_input_tensor_by_name(request, "IN").as_numpy() - mode_tensor = pb_utils.get_input_tensor_by_name(request, "MODE").as_numpy() - mode = mode_tensor.flat[0].decode("utf-8") - thread = threading.Thread( - target=self._response_thread, - args=(request.get_response_sender(), in_tensor, mode), - ) - thread.daemon = True - with self.inflight_thread_count_lck: - self.inflight_thread_count += 1 - thread.start() - return None - - def _response_thread(self, response_sender, in_tensor, mode): - # Send a normal response first. - out_tensor = pb_utils.Tensor("OUT", in_tensor) - response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) - response_sender.send(response) - - # Send an error response next. - error = pb_utils.TritonError("An error occurred during execution") - response = pb_utils.InferenceResponse(error=error) - - if mode == "MODE_ERROR_ONLY" or mode == "MODE_ERROR_WITH_DELAYED_FINAL": - response_sender.send(response) - elif mode == "MODE_ERROR_FINAL": - response_sender.send( - response, flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL - ) - else: - raise ValueError(f"Invalid mode: {mode}") - - # Send a delayed FINAL flag. - if mode == "MODE_ERROR_WITH_DELAYED_FINAL": - time.sleep(0.5) - response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) - - with self.inflight_thread_count_lck: - self.inflight_thread_count -= 1 - - def finalize(self): - inflight_threads = True - while inflight_threads: - with self.inflight_thread_count_lck: - inflight_threads = self.inflight_thread_count != 0 - if inflight_threads: - time.sleep(0.1) diff --git a/qa/python_models/identity_bf16/model.py b/qa/python_models/identity_bf16/model.py index d4ca2d250c..57756073b9 100644 --- a/qa/python_models/identity_bf16/model.py +++ b/qa/python_models/identity_bf16/model.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -71,7 +71,7 @@ def execute(self, requests): bf16_dlpack = input_tensor.to_dlpack() # OPTIONAL: The tensor can be converted to other dlpack-compatible - # frameworks like PyTorch with their dlpack utilities. + # frameworks like PyTorch and TensorFlow with their dlpack utilities. torch_tensor = torch.utils.dlpack.from_dlpack(bf16_dlpack) # When complete, convert back to a pb_utils.Tensor via DLPack. diff --git a/qa/python_models/join_add_sub/config.pbtxt b/qa/python_models/join_add_sub/config.pbtxt deleted file mode 100644 index f1abd51ec3..0000000000 --- a/qa/python_models/join_add_sub/config.pbtxt +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "join_add_sub" -backend: "python" -max_batch_size: 1 - -input [ - { - name: "INPUT0" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] -input [ - { - name: "INPUT1" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] -output [ - { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: [ 1 ] - } -] - -instance_group [{ kind: KIND_CPU }] diff --git a/qa/python_models/join_add_sub/model.py b/qa/python_models/join_add_sub/model.py deleted file mode 100644 index 79371cf38f..0000000000 --- a/qa/python_models/join_add_sub/model.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import json - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - def initialize(self, args): - self.model_config = model_config = json.loads(args["model_config"]) - - output0_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT0") - output1_config = pb_utils.get_output_config_by_name(model_config, "OUTPUT1") - - self.output0_dtype = pb_utils.triton_string_to_numpy( - output0_config["data_type"] - ) - self.output1_dtype = pb_utils.triton_string_to_numpy( - output1_config["data_type"] - ) - - def execute(self, requests): - output0_dtype = self.output0_dtype - output1_dtype = self.output1_dtype - - responses = [] - for request in requests: - in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") - in_1 = pb_utils.get_input_tensor_by_name(request, "INPUT1") - if ( - in_0.as_numpy().dtype.type is np.bytes_ - or in_0.as_numpy().dtype == np.object_ - ): - out_0, out_1 = ( - in_0.as_numpy().astype(np.int32) + in_1.as_numpy().astype(np.int32), - in_0.as_numpy().astype(np.int32) - in_1.as_numpy().astype(np.int32), - ) - else: - out_0, out_1 = ( - in_0.as_numpy() + in_1.as_numpy(), - in_0.as_numpy() - in_1.as_numpy(), - ) - - out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0.astype(output0_dtype)) - out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1.astype(output1_dtype)) - responses.append(pb_utils.InferenceResponse([out_tensor_0, out_tensor_1])) - return responses diff --git a/qa/python_models/python_version/model.py b/qa/python_models/python_version/model.py index d598681ae9..b1157ea50d 100644 --- a/qa/python_models/python_version/model.py +++ b/qa/python_models/python_version/model.py @@ -50,6 +50,8 @@ def initialize(self, args): self.model_config = args["model_config"] # This is to make sure that /bin/bash is not picking up # the wrong shared libraries after installing PyTorch. + # Tensorflow uses a shared library which is common with + # bash. os.system("/bin/bash --help") print( f"Python version is {sys.version_info.major}.{sys.version_info.minor}, NumPy version is {np.version.version}, and PyTorch version is {torch.__version__}", diff --git a/qa/python_models/response_sender_until_cancelled/config.pbtxt b/qa/python_models/response_sender_until_cancelled/config.pbtxt deleted file mode 100644 index 34a703db39..0000000000 --- a/qa/python_models/response_sender_until_cancelled/config.pbtxt +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -name: "response_sender_until_cancelled" -backend: "python" -model_transaction_policy { - decoupled: True -} - -input [ - { - name: "MAX_RESPONSE_COUNT" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "DELAY" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "INPUT" - data_type: TYPE_INT32 - dims: [ 1 ] - }, - { - name: "IGNORE_CANCEL" - data_type: TYPE_BOOL - dims: [ 1 ] - } -] -output [ - { - name: "OUTPUT" - data_type: TYPE_INT32 - dims: [ 1 ] - } -] - -instance_group [{ kind: KIND_CPU }] diff --git a/qa/python_models/response_sender_until_cancelled/model.py b/qa/python_models/response_sender_until_cancelled/model.py deleted file mode 100644 index c4ffe8dd32..0000000000 --- a/qa/python_models/response_sender_until_cancelled/model.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import time - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """This model will keep repeating the INPUT as the OUTPUT, - until the request is being cancelled or - the MAX_RESPONSE_COUNT has been reached. - """ - - def execute(self, requests): - request = requests[0] - - input = pb_utils.get_input_tensor_by_name(request, "INPUT").as_numpy() - max_response_count = pb_utils.get_input_tensor_by_name( - request, "MAX_RESPONSE_COUNT" - ).as_numpy()[0] - delay = pb_utils.get_input_tensor_by_name(request, "DELAY").as_numpy()[0] - ignore_cancel = pb_utils.get_input_tensor_by_name( - request, "IGNORE_CANCEL" - ).as_numpy()[0] - response_sender = request.get_response_sender() - - sent = 0 - while True: - if not ignore_cancel and request.is_cancelled(): - response = pb_utils.InferenceResponse( - error=pb_utils.TritonError( - message="request has been cancelled", - code=pb_utils.TritonError.CANCELLED, - ) - ) - response_sender.send( - response, flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL - ) - break - - output = pb_utils.Tensor("OUTPUT", np.array([input[0]], np.int32)) - response = pb_utils.InferenceResponse(output_tensors=[output]) - - if sent + 1 == max_response_count: - response_sender.send( - response, flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL - ) - break - else: - response_sender.send(response) - sent += 1 - time.sleep(delay / 1000) - - return None diff --git a/qa/python_models/torchvision/resnet50/model.py b/qa/python_models/torchvision/resnet50/model.py index 0e6405d302..6a31a77420 100644 --- a/qa/python_models/torchvision/resnet50/model.py +++ b/qa/python_models/torchvision/resnet50/model.py @@ -1,4 +1,4 @@ -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,6 @@ import torch import triton_python_backend_utils as pb_utils from torch.utils.dlpack import to_dlpack -from torchvision import models class TritonPythonModel: @@ -36,8 +35,15 @@ def initialize(self, args): This function initializes pre-trained ResNet50 model. """ self.device = "cuda" if args["model_instance_kind"] == "GPU" else "cpu" + # Avoid the "HTTP Error 403: rate limit exceeded" error + torch.hub._validate_not_a_forked_repo = lambda a, b, c: True + # Our tests currently depend on torchvision=0.14, + # to make sure `torch.hub` loads Resnet50 implementation + # compatible with torchvision=0.14, we need to provide tag self.model = ( - models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V2) + torch.hub.load( + "pytorch/vision:v0.14.1", "resnet50", weights="IMAGENET1K_V2" + ) .to(self.device) .eval() ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index de46e970e4..9445464ebc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -24,7 +24,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required (VERSION 3.31.8) +cmake_minimum_required (VERSION 3.18) project(tritonserverexe LANGUAGES C CXX) @@ -78,7 +78,7 @@ endif() # OpenTelemetry # -if (${TRITON_ENABLE_TRACING}) +if (NOT WIN32 AND ${TRITON_ENABLE_TRACING}) find_package(absl CONFIG REQUIRED) find_package(CURL CONFIG REQUIRED) find_package(nlohmann_json CONFIG REQUIRED) @@ -90,6 +90,19 @@ endif() # find_package(re2 REQUIRED) +option(TRITON_ENABLE_MYSQL_ODBC "Build tritonserver with MySQL ODBC connection pool (requires ODBC::ODBC / unixODBC)" OFF) +if(TRITON_ENABLE_MYSQL_ODBC) + if(UNIX AND NOT APPLE AND NOT WIN32) + message( + STATUS + "TRITON_ENABLE_MYSQL_ODBC: ensure unixODBC development packages are installed " + "(e.g. Debian/Ubuntu: unixodbc-dev; RHEL: unixODBC-devel) so CMake can find ODBC." + ) + endif() + find_package(ODBC REQUIRED) + message(STATUS "Using ODBC ${ODBC_VERSION}") +endif() + # # tritonserver executable # @@ -101,27 +114,72 @@ add_executable( main.cc shared_memory_manager.cc triton_signal.cc + database_config.cc classification.h common.h + database_config.h shared_memory_manager.h triton_signal.h ) -set_property(TARGET main PROPERTY OUTPUT_NAME tritonserver) +# On windows a *.lib file can be generated for a exe. When creating +# tritonserver.exe if we try to create tritonserver.lib it will fail +# because there is already a trtionserver.lib for tritonserver.dll, +# this causes the build to fail. To avoid we keep the build name as +# main.exe and then for windows after installing we rename it to +# tritonserver.exe (below in the install steps). +if (NOT WIN32) + set_property(TARGET main PROPERTY OUTPUT_NAME tritonserver) +endif() target_compile_features(main PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) -target_compile_options( - main - PRIVATE - -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Werror -) +if(WIN32) + message("Using MSVC as compiler, default target on Windows 10. " + "If the target system is not Windows 10, please update _WIN32_WINNT " + "to corresponding value.") + target_compile_options( + main + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + target_compile_definitions(main + PRIVATE + NOMINMAX) -# Dependency from common.h -target_link_libraries( - main - PRIVATE - b64 -) + # Dependency from common.h + find_library(B64_LIBRARY NAMES b64) + target_link_libraries( + main + PRIVATE + ${B64_LIBRARY} + ) + +else() + + target_compile_options( + main + PRIVATE + -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + + # Dependency from common.h + target_link_libraries( + main + PRIVATE + b64 + ) + endif() + +if(TRITON_ENABLE_MYSQL_ODBC) + target_sources( + main + PRIVATE + mysql_odbc_connection_pool.cc + mysql_odbc_connection_pool.h + ) + target_link_libraries(main PRIVATE ODBC::ODBC) + target_compile_definitions(main PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) +endif() set(LIB_DIR "lib") if(LINUX) @@ -148,6 +206,7 @@ target_link_libraries( triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-common-logging # from repo-common + triton-common-json # from repo-common (RapidJSON) triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) @@ -243,17 +302,29 @@ if(${TRITON_ENABLE_NVTX}) ) endif() # TRITON_ENABLE_NVTX -target_link_libraries( - main - PRIVATE - rt - dl -) +if (NOT WIN32) + target_link_libraries( + main + PRIVATE + rt + dl + ) +endif() # NOT WIN32 -install( - TARGETS main - RUNTIME DESTINATION bin -) +if (NOT WIN32) + install( + TARGETS main + RUNTIME DESTINATION bin + ) +else() + # See explanation above as to why we need to rename main.exe to + # tritonserver.exe as part of the install process on windows. + install( + PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/main.exe + DESTINATION bin + RENAME tritonserver.exe + ) +endif() if(${TRITON_ENABLE_GRPC}) # @@ -293,11 +364,13 @@ if(${TRITON_ENABLE_HTTP} list(APPEND HTTP_ENDPOINT_SRCS http_server.cc + multi_infer.cc orca_http.cc ) list(APPEND HTTP_ENDPOINT_HDRS http_server.h + http_server_macros.h orca_http.h ) @@ -330,11 +403,19 @@ if(${TRITON_ENABLE_HTTP} ) target_compile_features(http-endpoint-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) - target_compile_options( - http-endpoint-library - PRIVATE - -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wno-error=maybe-uninitialized -Werror - ) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options( + http-endpoint-library + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + http-endpoint-library + PRIVATE + -Wall -Wextra -Wno-unused-parameter -Wno-deprecated-declarations -Wno-error=maybe-uninitialized -Werror + ) + endif() set_target_properties( http-endpoint-library @@ -359,7 +440,10 @@ if(${TRITON_ENABLE_HTTP} PRIVATE $ ) - if (${TRITON_ENABLE_TRACING}) + # FIXME when Triton support of Opentelemetry is available on Windows + # add ${OPENTELEMETRY_CPP_INCLUDE_DIRS} to above target_include_directories + # JIRA DLIS-4786 + if (NOT WIN32 AND ${TRITON_ENABLE_TRACING}) target_link_libraries( http-endpoint-library PRIVATE tracing-library @@ -436,12 +520,23 @@ if(${TRITON_ENABLE_HTTP} ) endif() # TRITON_ENABLE_NVTX - target_link_libraries( - http-endpoint-library - PUBLIC - b64 - z - ) + if (WIN32) + find_library(B64_LIBRARY NAMES b64) + find_library(ZLIB_LIBRARY NAMES zlib) + target_link_libraries( + http-endpoint-library + PUBLIC + ${B64_LIBRARY} + ${ZLIB_LIBRARY} + ) + else() + target_link_libraries( + http-endpoint-library + PUBLIC + b64 + z + ) + endif() target_link_libraries( main @@ -462,15 +557,18 @@ if(${TRITON_ENABLE_TRACING}) ) target_compile_features(tracing-library PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) - target_include_directories( - tracing-library - PUBLIC ${OPENTELEMETRY_CPP_INCLUDE_DIRS} - ) + # FIXME: remove, when Windows support is added for Opentelemetry + if (NOT WIN32) + target_include_directories( + tracing-library + PUBLIC ${OPENTELEMETRY_CPP_INCLUDE_DIRS} + ) - target_link_libraries( - tracing-library - PUBLIC - ${OPENTELEMETRY_CPP_LIBRARIES}) + target_link_libraries( + tracing-library + PUBLIC + ${OPENTELEMETRY_CPP_LIBRARIES}) + endif() set_target_properties( tracing-library @@ -541,130 +639,99 @@ if(${TRITON_ENABLE_TRACING}) ) endif() # TRITON_ENABLE_TRACING -# -# simple -# -add_executable( - simple - simple.cc -) - -target_compile_features(simple PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) -target_compile_options( - simple - PRIVATE - -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror -) - -set_target_properties( - simple - PROPERTIES - POSITION_INDEPENDENT_CODE ON - SKIP_BUILD_RPATH TRUE - BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH_USE_LINK_PATH FALSE - INSTALL_RPATH "" -) - -target_link_libraries( - simple - PRIVATE - triton-common-async-work-queue # from repo-common - triton-common-error # from repo-common - triton-core-serverapi # from repo-core - triton-core-serverstub # from repo-core +if (NOT WIN32) + # + # simple + # + add_executable( + simple + simple.cc ) -if(${TRITON_ENABLE_GPU}) - target_compile_definitions( + target_compile_features(simple PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + message("Using MSVC as compiler, default target on Windows 10. " + "If the target system is not Windows 10, please update _WIN32_WINNT " + "to corresponding value.") + target_compile_options( + simple + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + simple + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + set_target_properties( simple - PRIVATE TRITON_ENABLE_GPU=1 - PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} + PROPERTIES + POSITION_INDEPENDENT_CODE ON + SKIP_BUILD_RPATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH FALSE + INSTALL_RPATH "" ) target_link_libraries( simple PRIVATE - CUDA::cudart - ) -endif() # TRITON_ENABLE_GPU - -install( - TARGETS simple - RUNTIME DESTINATION bin -) - -# -# multi_server example -# -add_executable( - multi_server - multi_server.cc -) - -target_compile_features(multi_server PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) -target_compile_options( - multi_server - PRIVATE - -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror -) - -set_target_properties( - multi_server - PROPERTIES - POSITION_INDEPENDENT_CODE ON - SKIP_BUILD_RPATH TRUE - BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH_USE_LINK_PATH FALSE - INSTALL_RPATH "" -) + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + ) -target_link_libraries( - multi_server - PRIVATE - triton-common-async-work-queue # from repo-common - triton-common-error # from repo-common - triton-core-serverapi # from repo-core - triton-core-serverstub # from repo-core - ) + if(${TRITON_ENABLE_GPU}) + target_compile_definitions( + simple + PRIVATE TRITON_ENABLE_GPU=1 + PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} + ) -if(${TRITON_ENABLE_GPU}) - target_compile_definitions( - multi_server - PRIVATE TRITON_ENABLE_GPU=1 - PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} - ) + target_link_libraries( + simple + PRIVATE + CUDA::cudart + ) + endif() # TRITON_ENABLE_GPU - target_link_libraries( - multi_server - PRIVATE - CUDA::cudart + install( + TARGETS simple + RUNTIME DESTINATION bin ) -endif() # TRITON_ENABLE_GPU -install( - TARGETS multi_server - RUNTIME DESTINATION bin -) - -if(${TRITON_ENABLE_GPU}) # - # memory_alloc example + # multi_server example # add_executable( - memory_alloc - memory_alloc.cc + multi_server + multi_server.cc ) - target_compile_features(memory_alloc PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) - target_compile_options( - memory_alloc - PRIVATE - -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror - ) + target_compile_features(multi_server PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + message("Using MSVC as compiler, default target on Windows 10. " + "If the target system is not Windows 10, please update _WIN32_WINNT " + "to corresponding value.") + target_compile_options( + multi_server + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + multi_server + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() set_target_properties( - memory_alloc + multi_server PROPERTIES POSITION_INDEPENDENT_CODE ON SKIP_BUILD_RPATH TRUE @@ -673,30 +740,157 @@ if(${TRITON_ENABLE_GPU}) INSTALL_RPATH "" ) - target_compile_definitions( - memory_alloc - PRIVATE TRITON_ENABLE_GPU=1 - PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} - ) - target_link_libraries( - memory_alloc + multi_server PRIVATE triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core - CUDA::cudart ) + if(${TRITON_ENABLE_GPU}) + target_compile_definitions( + multi_server + PRIVATE TRITON_ENABLE_GPU=1 + PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} + ) + + target_link_libraries( + multi_server + PRIVATE + CUDA::cudart + ) + endif() # TRITON_ENABLE_GPU + install( - TARGETS memory_alloc + TARGETS multi_server RUNTIME DESTINATION bin ) -endif() # TRITON_ENABLE_GPU -# tritonfrontend python package -add_subdirectory(python) + # + # transform (JSON -> RapidJSON, TRITONSERVER_Error reporting) + # + add_library(transform STATIC transform.cc transform.h) + + # Required when transform is linked into shared objects (e.g. py-bindings -> + # http-endpoint-library -> transform); otherwise relocations fail at link time. + set_target_properties( + transform + PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) + + target_compile_features(transform PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options( + transform + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + transform + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + target_link_libraries( + transform + PRIVATE + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-common-json # RapidJSON from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + ) + + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions(transform PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) + target_link_libraries(transform PRIVATE ODBC::ODBC) + endif() + + if((${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR + ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) AND + TARGET http-endpoint-library AND TARGET transform) + target_link_libraries(http-endpoint-library PRIVATE transform) + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions( + http-endpoint-library + PRIVATE TRITON_ENABLE_MYSQL_ODBC=1 + ) + endif() + endif() -add_subdirectory(test test) + if(${TRITON_ENABLE_GPU}) + # + # memory_alloc example + # + add_executable( + memory_alloc + memory_alloc.cc + ) + + target_compile_features(memory_alloc PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + message("Using MSVC as compiler, default target on Windows 10. " + "If the target system is not Windows 10, please update _WIN32_WINNT " + "to corresponding value.") + target_compile_options( + memory_alloc + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + memory_alloc + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + set_target_properties( + memory_alloc + PROPERTIES + POSITION_INDEPENDENT_CODE ON + SKIP_BUILD_RPATH TRUE + BUILD_WITH_INSTALL_RPATH TRUE + INSTALL_RPATH_USE_LINK_PATH FALSE + INSTALL_RPATH "" + ) + + target_compile_definitions( + memory_alloc + PRIVATE TRITON_ENABLE_GPU=1 + PRIVATE TRITON_MIN_COMPUTE_CAPABILITY=${TRITON_MIN_COMPUTE_CAPABILITY} + ) + + target_link_libraries( + memory_alloc + PRIVATE + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + CUDA::cudart + ) + + install( + TARGETS memory_alloc + RUNTIME DESTINATION bin + ) + endif() # TRITON_ENABLE_GPU +endif() # NOT WIN32 + +# DLIS-7292: Extend tritonfrontend to build for Windows +if (NOT WIN32) + # tritonfrontend python package + add_subdirectory(python) +endif (NOT WIN32) + +# Currently unit tests do not build for windows... +if ( NOT WIN32) + add_subdirectory(test test) +endif() # NOT WIN32 diff --git a/src/classification.cc b/src/classification.cc index ca50ceb6dc..2d8cd26b9e 100644 --- a/src/classification.cc +++ b/src/classification.cc @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2026, NVIDIA CORPORATION. All rights reserved. +// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -77,27 +77,8 @@ TopkClassifications( const TRITONSERVER_DataType datatype, const uint32_t req_class_count, std::vector* class_strs) { - const uint32_t dtype_byte_size = TRITONSERVER_DataTypeByteSize(datatype); - if (dtype_byte_size == 0) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - std::string( - std::string("class result not available for output due to " - "unsupported type '") + - std::string(TRITONSERVER_DataTypeString(datatype)) + "'") - .c_str()); - } - - const size_t element_cnt = byte_size / dtype_byte_size; - // Prevent pathological memory / CPU usage from unbounded classification - // outputs. - constexpr size_t kMaxClassificationElements = 1'000'000; - - if (element_cnt > kMaxClassificationElements) { - return TRITONSERVER_ErrorNew( - TRITONSERVER_ERROR_INVALID_ARG, - "classification output tensor too large"); - } + const size_t element_cnt = + byte_size / TRITONSERVER_DataTypeByteSize(datatype); switch (datatype) { case TRITONSERVER_TYPE_UINT8: diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc index f228bcaf77..1a829e0992 100644 --- a/src/command_line_parser.cc +++ b/src/command_line_parser.cc @@ -1,4 +1,4 @@ -// Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -298,7 +298,6 @@ enum TritonOptionId { OPTION_HTTP_ADDRESS, OPTION_HTTP_THREAD_COUNT, OPTION_HTTP_RESTRICTED_API, - OPTION_HTTP_MAX_INPUT_SIZE, #endif // TRITON_ENABLE_HTTP #if defined(TRITON_ENABLE_GRPC) OPTION_ALLOW_GRPC, @@ -306,7 +305,6 @@ enum TritonOptionId { OPTION_REUSE_GRPC_PORT, OPTION_GRPC_ADDRESS, OPTION_GRPC_HEADER_FORWARD_PATTERN, - OPTION_GRPC_INFER_THREAD_COUNT, OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE, OPTION_GRPC_MAX_RESPONSE_POOL_SIZE, OPTION_GRPC_USE_SSL, @@ -377,8 +375,7 @@ enum TritonOptionId { OPTION_HOST_POLICY, OPTION_MODEL_LOAD_GPU_LIMIT, OPTION_MODEL_NAMESPACING, - OPTION_ENABLE_PEER_ACCESS, - OPTION_ALLOW_CLIENT_SHM + OPTION_ENABLE_PEER_ACCESS }; void @@ -425,11 +422,11 @@ TritonParser::SetupOptions() "Specify the mode for model management. Options are \"none\", \"poll\" " "and \"explicit\". The default is \"none\". " "For \"none\", the server will load all models in the model " - "repository(s) at startup and will not make any changes to the loaded " + "repository(s) at startup and will not make any changes to the load " "models after that. For \"poll\", the server will poll the model " "repository(s) to detect changes and will load/unload models based on " "those changes. The poll rate is controlled by 'repository-poll-secs'. " - "For \"explicit\", model load and unload are initiated by using the " + "For \"explicit\", model load and unload is initiated by using the " "model control APIs, and only models specified with --load-model will " "be loaded at startup."}); model_repo_options_.push_back( @@ -499,16 +496,11 @@ TritonParser::SetupOptions() http_options_.push_back( {OPTION_HTTP_THREAD_COUNT, "http-thread-count", Option::ArgInt, "Number of threads handling HTTP requests."}); - http_options_.push_back( - {OPTION_HTTP_MAX_INPUT_SIZE, "http-max-input-size", Option::ArgInt, - ("Maximum allowed HTTP request input size in bytes. For compressed " - "requests, this also limits the decompressed size. Default is " + - std::to_string(HTTP_DEFAULT_MAX_INPUT_SIZE) + " bytes (64MB).")}); http_options_.push_back( {OPTION_HTTP_RESTRICTED_API, "http-restricted-api", ":=", "Specify restricted HTTP api setting. The format of this " - "flag is --http-restricted-api=:=. Where " + "flag is --http-restricted-api=,=. Where " " is a comma-separated list of apis to be restricted. " " will be additional header key to be checked when a HTTP request " "is received, and is the value expected to be matched." @@ -538,10 +530,6 @@ TritonParser::SetupOptions() Option::ArgStr, "The regular expression pattern that will be used for forwarding GRPC " "headers as inference request parameters."}); - grpc_options_.push_back( - {OPTION_GRPC_INFER_THREAD_COUNT, "grpc-infer-thread-count", - Option::ArgInt, - "The number of gRPC inference handler threads. Default is 2."}); grpc_options_.push_back( {OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE, "grpc-infer-allocation-pool-size", Option::ArgInt, @@ -632,7 +620,7 @@ TritonParser::SetupOptions() {OPTION_GRPC_RESTRICTED_PROTOCOL, "grpc-restricted-protocol", ":=", "Specify restricted GRPC protocol setting. The format of this " - "flag is --grpc-restricted-protocol=:=. Where " + "flag is --grpc-restricted-protocol=,=. Where " " is a comma-separated list of protocols to be restricted. " " will be additional header key to be checked when a GRPC request " "is received, and is the value expected to be matched." @@ -849,13 +837,6 @@ TritonParser::SetupOptions() "limit, the load will be rejected. If not specified, the limit will " "not be set."}); - shared_memory_options_.push_back( - {OPTION_ALLOW_CLIENT_SHM, "allow-client-shm", Option::ArgBool, - "Allow clients to register/unregister and use shared memory regions " - "(both CPU system shared memory and GPU CUDA IPC shared memory) for " - "inference inputs and outputs. Internal shared memory used by backends " - "such as the Python backend is unaffected. Default is false."}); - backend_options_.push_back( {OPTION_BACKEND_DIR, "backend-directory", Option::ArgStr, "The global directory searched for backend shared libraries. Default is " @@ -938,7 +919,6 @@ TritonParser::SetupOptionGroups() option_groups_.emplace_back("Rate Limiter", rate_limiter_options_); option_groups_.emplace_back( "Memory/Device Management", memory_device_options_); - option_groups_.emplace_back("Shared Memory", shared_memory_options_); option_groups_.emplace_back("DEPRECATED", deprecated_options_); } @@ -1408,15 +1388,6 @@ TritonParser::Parse(int argc, char** argv) case OPTION_HTTP_THREAD_COUNT: lparams.http_thread_cnt_ = ParseOption(optarg); break; - case OPTION_HTTP_MAX_INPUT_SIZE: { - int64_t temp_input_size = ParseOption(optarg); - if (temp_input_size <= 0) { - throw ParseException( - "Error: --http-max-input-size must be greater than 0."); - } - lparams.http_max_input_size_ = temp_input_size; - break; - } case OPTION_HTTP_RESTRICTED_API: ParseRestrictedFeatureOption( optarg, long_options[option_index].name, "", "api", @@ -1470,15 +1441,6 @@ TritonParser::Parse(int argc, char** argv) case OPTION_GRPC_ADDRESS: lgrpc_options.socket_.address_ = optarg; break; - case OPTION_GRPC_INFER_THREAD_COUNT: - lgrpc_options.infer_thread_count_ = ParseOption(optarg); - if (lgrpc_options.infer_thread_count_ < 2 || - lgrpc_options.infer_thread_count_ > 128) { - throw ParseException( - "invalid argument for --grpc_infer_thread_count. Must be in " - "the range 2 to 128."); - } - break; case OPTION_GRPC_INFER_ALLOCATION_POOL_SIZE: lgrpc_options.infer_allocation_pool_size_ = ParseOption(optarg); break; @@ -1790,9 +1752,6 @@ TritonParser::Parse(int argc, char** argv) case OPTION_ENABLE_PEER_ACCESS: lparams.enable_peer_access_ = ParseOption(optarg); break; - case OPTION_ALLOW_CLIENT_SHM: - lparams.allow_client_shm_ = ParseOption(optarg); - break; } } catch (const ParseException& pe) { diff --git a/src/command_line_parser.h b/src/command_line_parser.h index c7effe8960..762ee87b6d 100644 --- a/src/command_line_parser.h +++ b/src/command_line_parser.h @@ -1,4 +1,4 @@ -// Copyright 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -35,7 +35,6 @@ #include #include -#include "common.h" #include "restricted_features.h" #include "triton/common/logging.h" #include "triton/core/tritonserver.h" @@ -198,8 +197,6 @@ struct TritonServerParameters { // The number of threads to initialize for the HTTP front-end. int http_thread_cnt_{8}; RestrictedFeatures http_restricted_apis_{}; - // Default value 64MB - size_t http_max_input_size_{HTTP_DEFAULT_MAX_INPUT_SIZE}; #endif // TRITON_ENABLE_HTTP #ifdef TRITON_ENABLE_GRPC @@ -240,9 +237,6 @@ struct TritonServerParameters { std::string vertex_ai_default_model_{}; #endif // TRITON_ENABLE_VERTEX_AI - // Shared memory access control - bool allow_client_shm_{false}; - // [FIXME] who should call this function? void CheckPortCollision(); using ManagedTritonServerOptionPtr = std::unique_ptr< @@ -353,7 +347,6 @@ class TritonParser { std::vector