diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yml similarity index 89% rename from .github/workflows/pre-commit.yaml rename to .github/workflows/pre-commit.yml index 6c33d435c9..15d0b68685 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yml @@ -1,4 +1,4 @@ -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,15 +31,15 @@ on: jobs: pre-commit: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v5.0.0 with: fetch-depth: 2 - name: Get modified files id: modified-files run: echo "modified_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT - - uses: actions/setup-python@v3 - - uses: pre-commit/action@v3.0.0 + - uses: actions/setup-python@v6.0.0 + - uses: pre-commit/action@v3.0.1 with: extra_args: --files ${{ steps.modified-files.outputs.modified_files }} diff --git a/.gitignore b/.gitignore index bce94b6830..01abc6eaab 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,12 @@ test_results.txt artifacts cprofile *.prof +.venv +**/.venv # Test exclusions qa/L0_openai/openai tensorrtllm_models +tensorrtllm_mistral_models/ custom_tokenizer +replace-artifacts/ \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 663a36d631..cd6320fe0d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. repos: -- repo: https://github.com/timothycrosley/isort +- repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: - id: isort @@ -36,7 +36,7 @@ repos: - id: black types_or: [python, cython] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.3.0 hooks: - id: flake8 args: [--max-line-length=88, --select=C,E,F,W,B,B950, --extend-ignore = E203,E501] @@ -57,7 +57,7 @@ repos: # More details about these pre-commit hooks here: # https://pre-commit.com/hooks.html - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d03d85665..de1e229c2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,21 @@ cmake_minimum_required(VERSION 3.18) +# CMake 4.0+ rejects projects that call cmake_minimum_required(VERSION < 3.5). +# The fetched third_party repo builds libevent 2.1.12 via ExternalProject using a +# separate CMake invocation; that step does not inherit -D variables from the top +# level unless they are also in the environment. If configure fails inside libevent +# with "Compatibility with CMake < 3.5 has been removed", use either: +# export CMAKE_POLICY_VERSION_MINIMUM=3.5 # before cmake AND cmake --build +# or install/use CMake 3.28.x–3.31.x for this build. +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0") + message( + STATUS + "CMake ${CMAKE_VERSION}: for libevent/third_party with CMake 4.x, export " + "CMAKE_POLICY_VERSION_MINIMUM=3.5 in your shell before configure and build " + "(see docs/customization_guide/build.md).") +endif() + project(tritonserver LANGUAGES C CXX) include(CMakeDependentOption) @@ -65,6 +80,10 @@ option(TRITON_ENABLE_GCS "Include GCS Filesystem support in server" OFF) option(TRITON_ENABLE_S3 "Include S3 Filesystem support in server" OFF) option(TRITON_ENABLE_AZURE_STORAGE "Include Azure Storage Filesystem support in server" OFF) +option(TRITON_ENABLE_MYSQL_ODBC + "Enable MySQL ODBC connection pool in tritonserver (requires unixODBC / ODBC dev package)" + OFF) + # Need to know if TensorRT is available when building unit tests option(TRITON_ENABLE_TENSORRT "Include TensorRT backend in server" OFF) @@ -261,6 +280,7 @@ ExternalProject_Add(triton-server -DTRITON_ENABLE_S3:BOOL=${TRITON_ENABLE_S3} -DTRITON_ENABLE_TENSORRT:BOOL=${TRITON_ENABLE_TENSORRT} -DTRITON_ENABLE_ENSEMBLE:BOOL=${TRITON_ENABLE_ENSEMBLE} + -DTRITON_ENABLE_MYSQL_ODBC:BOOL=${TRITON_ENABLE_MYSQL_ODBC} -DTRITON_MIN_CXX_STANDARD:STRING=${TRITON_MIN_CXX_STANDARD} -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX:PATH=${TRITON_INSTALL_PREFIX} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..27454c0d38 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,109 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Derivative image: start from an official Triton server image and replace only +# the tritonserver executable and libtritonserver.so. Backends, Python wheels, +# and the rest of the filesystem stay unchanged from the base image. +# +# After a local CMake install (same layout as your -DCMAKE_INSTALL_PREFIX): +# install/bin/tritonserver +# install/lib/libtritonserver.so +# +# Prepare artifacts (from repository root): +# mkdir -p replace-artifacts +# cp install/bin/tritonserver replace-artifacts/ +# cp install/lib/libtritonserver.so replace-artifacts/ +# cp /etc/odbc.ini replace-artifacts/odbc.ini # optional if databaseIp is set (DSN unused) +# Run `odbcinst -q -d` inside the built image to see the exact ODBC driver name for +# optional JSON field "odbcDriverName" if the default fails. +# +# Build (from repository root; match your Triton tag, e.g. r25.03): +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 \ +# -t tritonserver:25.03-custom . +# +# CPU-only base example: +# docker build \ +# --build-arg BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min \ +# -t tritonserver:25.03-custom-cpu . +# +# If your base image stores libtritonserver.so under lib64: +# --build-arg TRITON_LIB_SUBDIR=lib64 +# +# Ubuntu 24.04 (noble) base images: use Connector package for 24.04, e.g.: +# --build-arg MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu24.04 +# +# Run (mount a model repository and optionally override DM config): +# docker run --rm --gpus=all \ +# -p8000:8000 -p8001:8001 -p8002:8002 \ +# -v /path/to/model_repo:/models:ro \ +# -v /path/to/your-triton-dmconfig.json:/etc/triton-dmconfig.json:ro \ +# tritonserver:25.03-custom \ +# tritonserver --model-repository=/models +# +# For CPU-only, drop --gpus=all and use a CPU/min base image. + +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3 + +FROM ${BASE_IMAGE} + +ARG TRITON_INSTALL_PREFIX=/opt/tritonserver +ARG TRITON_LIB_SUBDIR=lib + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +# unixODBC + official MySQL Connector/ODBC .deb (libmyodbc8 is often only in Ubuntu Universe +# or missing on minimal images). Override MYSQL_ODBC_DEB_* for noble/arm64, etc. +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +# mysql-connector-odbc registers Driver=/usr/lib/.../odbc/libmyodbc9w.so in +# /etc/odbcinst.ini, but Ubuntu/Debian often install the .so under +# /usr/lib//odbc/ only. unixODBC then fails with "Can't open lib +# '/usr/lib/odbc/libmyodbc9w.so'". Symlink all libmyodbc*.so into /usr/lib/odbc/. +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +# Optional DSN file (only used when databaseIp is empty in triton-dmconfig.json). +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +# Default DM database metadata (override at runtime with -v ...:/etc/triton-dmconfig.json:ro) +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json + +# Paths relative to the build context (repository root when building with ".") +COPY replace-artifacts/tritonserver ${TRITON_INSTALL_PREFIX}/bin/tritonserver +COPY replace-artifacts/libtritonserver.so \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + +# Match ownership used by generated Triton Dockerfiles (triton-server uid) +RUN chown 1000:1000 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so \ + && chmod 755 \ + ${TRITON_INSTALL_PREFIX}/bin/tritonserver \ + ${TRITON_INSTALL_PREFIX}/${TRITON_LIB_SUBDIR}/libtritonserver.so + diff --git a/Dockerfile.QA b/Dockerfile.QA index 2205e0c116..7406cf2345 100644 --- a/Dockerfile.QA +++ b/Dockerfile.QA @@ -61,6 +61,7 @@ RUN apt-get update && \ python3-pip \ python3-wheel \ python3-setuptools \ + python3-venv \ rapidjson-dev \ software-properties-common && \ rm -rf /var/lib/apt/lists/* @@ -74,12 +75,19 @@ RUN apt update -q=2 \ && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* # Add inception_graphdef model to example repo +# FIXME: This should be changed to using the fetch_models.sh script +# in order to ensure the public facing docs are up-to-date. WORKDIR /workspace/docs/examples/model_repository -RUN mkdir -p inception_graphdef/1 && \ - wget -O ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb.tar.gz \ - https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \ - (cd ${TRITONTMP_DIR} && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \ - mv ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb inception_graphdef/1/model.graphdef +RUN mkdir -p model_repository/inception_onnx/1 && \ + wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \ + https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \ + (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \ + python3 -m venv tf2onnx && \ + source ./tf2onnx/bin/activate && \ + pip3 install "numpy<2" tensorflow tf2onnx && \ + python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 && \ + deactivate && \ + mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx # Update the qa/ directory with test executables, models, etc. WORKDIR /workspace @@ -109,7 +117,7 @@ RUN mkdir -p qa/common && \ cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \ - cp -r docs/examples/model_repository/inception_graphdef qa/L0_grpc/models && \ + cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \ mkdir qa/L0_grpc_state_cleanup/models && \ cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \ mkdir qa/L0_http/models && \ @@ -118,7 +126,7 @@ RUN mkdir -p qa/common && \ cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \ cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \ - cp -r docs/examples/model_repository/inception_graphdef qa/L0_http/models && \ + cp -r docs/examples/model_repository/inception_onnx qa/L0_grpc/models && \ mkdir qa/L0_https/models && \ cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \ mkdir qa/L0_secure_grpc/models && \ @@ -149,21 +157,20 @@ RUN mkdir -p qa/common && \ cp bin/triton_json_test qa/L0_json/. && \ cp bin/backend_output_detail_test qa/L0_backend_output_detail/. && \ cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/. && \ - cp bin/input_byte_size_test qa/L0_input_validation/. && \ - cp -r docs/examples/model_repository/simple_identity qa/L0_input_validation/models + cp bin/input_byte_size_test qa/L0_input_validation/. RUN mkdir -p qa/pkgs && \ cp python/triton*.whl qa/pkgs/. && \ cp -rf python/test/. qa/L0_python_api/. RUN mkdir -p qa/L0_simple_ensemble/models/simple/1 && \ - cp docs/examples/model_repository/simple/1/model.graphdef \ + cp docs/examples/model_repository/simple/1/model.onnx \ qa/L0_simple_ensemble/models/simple/1/. && \ mkdir -p qa/L0_simple_ensemble/models/simple/2 && \ - cp docs/examples/model_repository/simple/1/model.graphdef \ + cp docs/examples/model_repository/simple/1/model.onnx \ qa/L0_simple_ensemble/models/simple/2/. && \ mkdir -p qa/L0_socket/models/simple/1 && \ - cp docs/examples/model_repository/simple/1/model.graphdef \ + cp docs/examples/model_repository/simple/1/model.onnx \ qa/L0_socket/models/simple/1/. RUN mkdir -p qa/L0_backend_identity/models && \ diff --git a/Dockerfile.pubmatic-bt-tritonserver b/Dockerfile.pubmatic-bt-tritonserver new file mode 100644 index 0000000000..340c3b27b5 --- /dev/null +++ b/Dockerfile.pubmatic-bt-tritonserver @@ -0,0 +1,52 @@ +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Extend the image produced by build.py (`docker build -t tritonserver`) with +# MySQL ODBC runtime, driver path symlinks for unixODBC, and config files. +# Build context: repository root (same as build.py final docker build). +# +# Prerequisites: +# - Image `tritonserver` must exist. +# - replace-artifacts/odbc.ini +# - replace-artifacts/triton-dmconfig.json +# +# Example: +# docker build -f Dockerfile.pubmatic-bt-tritonserver -t pubmatic-bt-tritonserver . + +FROM tritonserver + +ARG MYSQL_ODBC_DEB_VERSION=9.7.0-1ubuntu22.04 +ARG MYSQL_ODBC_DEB_ARCH=amd64 + +USER root + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + unixodbc \ + odbcinst; \ + DEB="mysql-connector-odbc_${MYSQL_ODBC_DEB_VERSION}_${MYSQL_ODBC_DEB_ARCH}.deb"; \ + curl -fsSL -o "/tmp/${DEB}" \ + "https://repo.mysql.com/apt/ubuntu/pool/mysql-tools/m/mysql-connector-odbc/${DEB}"; \ + apt-get install -y "/tmp/${DEB}" || apt-get -fy install; \ + rm -f "/tmp/${DEB}"; \ + rm -rf /var/lib/apt/lists/* + +RUN set -eux; \ + mkdir -p /usr/lib/odbc; \ + for f in \ + /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*.so \ + /usr/lib/aarch64-linux-gnu/odbc/libmyodbc*.so; \ + do \ + if [ -f "${f}" ]; then \ + ln -sf "${f}" "/usr/lib/odbc/$(basename "${f}")"; \ + fi; \ + done; \ + test -f /usr/lib/odbc/libmyodbc9w.so + +COPY replace-artifacts/odbc.ini /etc/odbc.ini +RUN chmod 644 /etc/odbc.ini + +COPY replace-artifacts/triton-dmconfig.json /etc/triton-dmconfig.json +RUN chmod 644 /etc/triton-dmconfig.json diff --git a/Dockerfile.sdk b/Dockerfile.sdk index f28d7f710e..de6b12df5e 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.02-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.03-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo diff --git a/README.md b/README.md index f62a1a176e..65d4918be5 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Major features include: - Provides [Backend API](https://github.com/triton-inference-server/backend) that allows adding custom backends and pre/post processing operations - Supports writing custom backends in python, a.k.a. - [Python-based backends.](https://github.com/triton-inference-server/backend/blob/main/docs/python_based_backends.md#python-based-backends) + [Python-based backends.](https://github.com/triton-inference-server/backend/blob/r25.03/docs/python_based_backends.md#python-based-backends) - Model pipelines using [Ensembling](docs/user_guide/architecture.md#ensemble-models) or [Business Logic Scripting @@ -84,17 +84,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.02 https://github.com/triton-inference-server/server.git +git clone -b r25.03 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.02-py3 tritonserver --model-repository=/models +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.03-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.02-py3-sdk -/workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.03-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': @@ -168,10 +167,10 @@ configuration](docs/user_guide/model_configuration.md) for the model. [Python](https://github.com/triton-inference-server/python_backend), and more - Not all the above backends are supported on every platform supported by Triton. Look at the - [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/main/docs/backend_platform_support_matrix.md) + [Backend-Platform Support Matrix](https://github.com/triton-inference-server/backend/blob/r25.03/docs/backend_platform_support_matrix.md) to learn which backends are supported on your target platform. - Learn how to [optimize performance](docs/user_guide/optimization.md) using the - [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/main/README.md) + [Performance Analyzer](https://github.com/triton-inference-server/perf_analyzer/blob/r25.03/README.md) and [Model Analyzer](https://github.com/triton-inference-server/model_analyzer) - Learn how to [manage loading and unloading models](docs/user_guide/model_management.md) in @@ -185,14 +184,14 @@ A Triton *client* application sends inference and other requests to Triton. The [Python and C++ client libraries](https://github.com/triton-inference-server/client) provide APIs to simplify this communication. -- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/main/src/c%2B%2B/examples), - [Python](https://github.com/triton-inference-server/client/blob/main/src/python/examples), - and [Java](https://github.com/triton-inference-server/client/blob/main/src/java/src/main/java/triton/client/examples) +- Review client examples for [C++](https://github.com/triton-inference-server/client/blob/r25.03/src/c%2B%2B/examples), + [Python](https://github.com/triton-inference-server/client/blob/r25.03/src/python/examples), + and [Java](https://github.com/triton-inference-server/client/blob/r25.03/src/java/src/main/java/triton/client/examples) - Configure [HTTP](https://github.com/triton-inference-server/client#http-options) and [gRPC](https://github.com/triton-inference-server/client#grpc-options) client options - Send input data (e.g. a jpeg image) directly to Triton in the [body of an HTTP - request without any additional metadata](https://github.com/triton-inference-server/server/blob/main/docs/protocol/extension_binary_data.md#raw-binary-request) + request without any additional metadata](https://github.com/triton-inference-server/server/blob/r25.03/docs/protocol/extension_binary_data.md#raw-binary-request) ### Extend Triton @@ -201,7 +200,7 @@ designed for modularity and flexibility - [Customize Triton Inference Server container](docs/customization_guide/compose.md) for your use case - [Create custom backends](https://github.com/triton-inference-server/backend) - in either [C/C++](https://github.com/triton-inference-server/backend/blob/main/README.md#triton-backend-api) + in either [C/C++](https://github.com/triton-inference-server/backend/blob/r25.03/README.md#triton-backend-api) or [Python](https://github.com/triton-inference-server/python_backend) - Create [decoupled backends and models](docs/user_guide/decoupled_models.md) that can send multiple responses for a request or not send any responses for a request diff --git a/TRITON_VERSION b/TRITON_VERSION index 4621559b25..5f46e11eed 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.56.0dev +2.56.0 diff --git a/build.py b/build.py index e35e910d98..6ea96f5218 100755 --- a/build.py +++ b/build.py @@ -71,14 +71,14 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.56.0dev", - "triton_container_version": "25.03dev", - "upstream_container_version": "25.02", - "ort_version": "1.20.1", + "release_version": "2.56.0", + "triton_container_version": "25.03", + "upstream_container_version": "25.03", + "ort_version": "1.21.0", "ort_openvino_version": "2025.0.0", "standalone_openvino_version": "2025.0.0", "dcgm_version": "3.3.6", - "vllm_version": "0.7.0", + "vllm_version": "0.7.3", "rhel_py_version": "3.12.3", } @@ -498,6 +498,7 @@ def core_cmake_args(components, backends, cmake_dir, install_dir): cargs.append(cmake_core_enable("TRITON_ENABLE_ENSEMBLE", "ensemble" in backends)) cargs.append(cmake_core_enable("TRITON_ENABLE_TENSORRT", "tensorrt" in backends)) + cargs.append(cmake_core_enable("TRITON_ENABLE_MYSQL_ODBC", FLAGS.enable_mysql_odbc)) cargs += cmake_core_extra_args() cargs.append(cmake_dir) @@ -562,8 +563,6 @@ def backend_cmake_args(images, components, be, install_dir, library_paths): args = onnxruntime_cmake_args(images, library_paths) elif be == "openvino": args = openvino_cmake_args() - elif be == "tensorflow": - args = tensorflow_cmake_args(images, library_paths) elif be == "python": args = python_cmake_args() elif be == "dali": @@ -795,23 +794,6 @@ def tensorrt_cmake_args(): return cargs -def tensorflow_cmake_args(images, library_paths): - backend_name = "tensorflow" - extra_args = [] - - # If a specific TF image is specified use it, otherwise pull from NGC. - if backend_name in images: - image = images[backend_name] - else: - image = "nvcr.io/nvidia/tensorflow:{}-tf2-py3".format( - FLAGS.upstream_container_version - ) - extra_args = [ - cmake_backend_arg(backend_name, "TRITON_TENSORFLOW_DOCKER_IMAGE", None, image) - ] - return extra_args - - def dali_cmake_args(): return [ cmake_backend_enable("dali", "TRITON_DALI_SKIP_DOWNLOAD", False), @@ -921,6 +903,12 @@ def install_dcgm_libraries(dcgm_version, target_machine): def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): + buildbase_odbc_layer = "" + if FLAGS.enable_mysql_odbc: + buildbase_odbc_layer = """ +RUN yum install -y unixODBC-devel +""" + df = """ ARG TRITON_VERSION={} ARG TRITON_CONTAINER_VERSION={} @@ -978,6 +966,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): xz-devel \\ zlib-devel """ + df += buildbase_odbc_layer if os.getenv("CCACHE_REMOTE_ONLY") and os.getenv("CCACHE_REMOTE_STORAGE"): df += """ RUN curl -k -s -L https://github.com/ccache/ccache/archive/refs/tags/v4.10.2.tar.gz -o /tmp/ccache.tar.gz \\ @@ -1070,6 +1059,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): SHELL ["cmd", "/S", "/C"] """ else: + mysql_odbc_line = ( + " unixodbc-dev \\\n" if FLAGS.enable_mysql_odbc else "" + ) df += """ # Ensure apt-get won't prompt for selecting options ENV DEBIAN_FRONTEND=noninteractive @@ -1120,7 +1112,9 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): libarchive-dev \\ libxml2-dev \\ libnuma-dev \\ - wget \\ +""" + df += mysql_odbc_line + df += """ wget \\ && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ @@ -1233,10 +1227,10 @@ def create_dockerfile_linux( argmap["BASE_IMAGE"], ) - # PyTorch and TensorFlow backends need extra CUDA and other + # PyTorch backends need extra CUDA and other # dependencies during runtime that are missing in the CPU-only base container. # These dependencies must be copied from the Triton Min image. - if not FLAGS.enable_gpu and (("pytorch" in backends) or ("tensorflow" in backends)): + if not FLAGS.enable_gpu and ("pytorch" in backends): df += """ ############################################################################ ## Triton Min image @@ -1602,10 +1596,10 @@ def add_cpu_libs_to_linux_dockerfile(backends, target_machine): cuda_arch=cuda_arch, libs_arch=libs_arch ) - if ("pytorch" in backends) or ("tensorflow" in backends): - # Add NCCL dependency for tensorflow/pytorch backend. + if "pytorch" in backends: + # Add NCCL dependency for pytorch backend. # Note: Even though the build is CPU-only, the version of - # tensorflow/pytorch we are using depends upon the NCCL library. + # pytorch we are using depends upon the NCCL library. # Since this dependency is not present in the ubuntu base image, # we must copy it from the Triton min container ourselves. df += """ @@ -1720,11 +1714,10 @@ def create_build_dockerfiles( } # For CPU-only image we need to copy some cuda libraries and dependencies - # since we are using PyTorch and TensorFlow containers that - # are not CPU-only. + # since we are using PyTorch containers that are not CPU-only. if ( not FLAGS.enable_gpu - and (("pytorch" in backends) or ("tensorflow" in backends)) + and ("pytorch" in backends) and (target_platform() != "windows") ): if "gpu-base" in images: @@ -1924,6 +1917,34 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ docker_script.cwd(THIS_SCRIPT_DIR) docker_script.cmd(finalargs, check_exitcode=True) + if ( + FLAGS.pubmatic_bt_tritonserver_image + and target_platform() != "windows" + and target_platform() != "rhel" + ): + docker_script.blankln() + docker_script.commentln(8) + docker_script.comment( + "Pubmatic BT image: extend tritonserver with MySQL ODBC and config files" + ) + docker_script.comment( + "Uses Dockerfile.pubmatic-bt-tritonserver; requires replace-artifacts/" + ) + docker_script.comment() + pubmatic_args = [ + "docker", + "build", + "-t", + "pubmatic-bt-tritonserver", + "-f", + os.path.join( + THIS_SCRIPT_DIR, "Dockerfile.pubmatic-bt-tritonserver" + ), + ".", + ] + docker_script.cwd(THIS_SCRIPT_DIR) + docker_script.cmd(pubmatic_args, check_exitcode=True) + # # CI base image... tritonserver_cibase # @@ -2351,7 +2372,6 @@ def enable_all(): "identity", "square", "repeat", - "tensorflow", "onnxruntime", "python", "dali", @@ -2586,7 +2606,7 @@ def enable_all(): "--image", action="append", required=False, - help='Use specified Docker image in build as ,. can be "base", "gpu-base", "tensorflow", or "pytorch".', + help='Use specified Docker image in build as ,. can be "base", "gpu-base", or "pytorch".', ) parser.add_argument( @@ -2637,6 +2657,20 @@ def enable_all(): required=False, help="Enable ARM MALI GPU support.", ) + parser.add_argument( + "--enable-mysql-odbc", + action="store_true", + required=False, + help="Build tritonserver with MySQL ODBC connection pool. For host builds install unixodbc-dev (Debian/Ubuntu) or unixODBC-devel (RHEL). Container builds add these when this flag is set.", + ) + parser.add_argument( + "--pubmatic-bt-tritonserver-image", + action="store_true", + required=False, + help='After building image "tritonserver", also build "pubmatic-bt-tritonserver" ' + "(Dockerfile.pubmatic-bt-tritonserver: ODBC driver + replace-artifacts/odbc.ini and " + "replace-artifacts/triton-dmconfig.json). Ubuntu/Debian-based container builds only.", + ) parser.add_argument( "--min-compute-capability", type=str, @@ -2887,12 +2921,6 @@ def enable_all(): parts = be.split(":") if len(parts) == 1: parts.append(default_repo_tag) - if parts[0] == "tensorflow1": - fail( - "Starting from Triton version 23.04, support for TensorFlow 1 has been discontinued. Please switch to Tensorflow 2." - ) - if parts[0] == "tensorflow2": - parts[0] = "tensorflow" log('backend "{}" at tag/branch "{}"'.format(parts[0], parts[1])) backends[parts[0]] = parts[1] @@ -2939,13 +2967,10 @@ def enable_all(): len(parts) != 2, "--image must specify ," ) fail_if( - parts[0] - not in ["base", "gpu-base", "pytorch", "tensorflow", "tensorflow2"], + parts[0] not in ["base", "gpu-base", "pytorch"], "unsupported value for --image", ) log('image "{}": "{}"'.format(parts[0], parts[1])) - if parts[0] == "tensorflow2": - parts[0] = "tensorflow" images[parts[0]] = parts[1] # Initialize map of library paths for each backend. @@ -2954,8 +2979,6 @@ def enable_all(): parts = lpath.split(":") if len(parts) == 2: log('backend "{}" library path "{}"'.format(parts[0], parts[1])) - if parts[0] == "tensorflow2": - parts[0] = "tensorflow" library_paths[parts[0]] = parts[1] # Parse any explicitly specified cmake arguments diff --git a/compose.py b/compose.py index 1ade3bef46..88c6b15bf3 100755 --- a/compose.py +++ b/compose.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -71,14 +71,10 @@ def start_dockerfile(ddir, images, argmap, dockerfile_name, backends): argmap["TRITON_VERSION"], argmap["TRITON_CONTAINER_VERSION"], images["full"] ) - # PyTorch, TensorFlow backends need extra CUDA and other + # PyTorch backends need extra CUDA and other # dependencies during runtime that are missing in the CPU-only base container. # These dependencies must be copied from the Triton Min image. - if not FLAGS.enable_gpu and ( - ("pytorch" in backends) - or ("tensorflow" in backends) - or ("tensorflow2" in backends) - ): + if not FLAGS.enable_gpu and "pytorch" in backends: df += """ FROM {} AS min_container @@ -302,7 +298,7 @@ def create_argmap(images, skip_pull): dcgm_ver = re.search("DCGM_VERSION=([\S]{4,}) ", vars) dcgm_version = "" if dcgm_ver is None: - dcgm_version = "2.2.3" + dcgm_version = "3.3.6" log( "WARNING: DCGM version not found from image, installing the earlierst version {}".format( dcgm_version @@ -406,7 +402,7 @@ def create_argmap(images, skip_pull): ',. can be "min", "gpu-min" ' 'or "full". Both "min" and "full" need to be specified at the same time.' 'This will override "--container-version". "gpu-min" is needed for ' - "CPU-only container to copy TensorFlow and PyTorch deps.", + "CPU-only container to copy PyTorch deps.", ) parser.add_argument( "--enable-gpu", @@ -504,13 +500,9 @@ def create_argmap(images, skip_pull): fail_if(len(images) < 2, "Need to specify both 'full' and 'min' images if at all") # For CPU-only image we need to copy some cuda libraries and dependencies - # since we are using PyTorch, TensorFlow 1, TensorFlow 2 containers that + # since we are using PyTorch containers that # are not CPU-only. - if ( - ("pytorch" in FLAGS.backend) - or ("tensorflow" in FLAGS.backend) - or ("tensorflow2" in FLAGS.backend) - ) and ("gpu-min" not in images): + if ("pytorch" in FLAGS.backend) and ("gpu-min" not in images): images["gpu-min"] = "nvcr.io/nvidia/tritonserver:{}-py3-min".format( FLAGS.container_version ) diff --git a/config/database_config.sample.json b/config/database_config.sample.json new file mode 100644 index 0000000000..db12e70189 --- /dev/null +++ b/config/database_config.sample.json @@ -0,0 +1,13 @@ +{ + "databaseIp" : "10.0.0.1", + "databasePort" : 3306, + "odbcDriverName" : "MySQL ODBC 9.7 Unicode Driver", + "primaryDSNName" : "PrimaryCentralisedMySQL", + "secondaryDSNName" : "SecondaryCentralisedMySQL", + "dsnUserName" : "kdbuser", + "dsnUserPassword" : "REPLACE_WITH_SECRET", + "queryRetryCount": 3, + "dcId": 1, + "minPoolConnections": 2, + "maxPoolConnections": 5 +} diff --git a/deploy/alibaba-cloud/README.md b/deploy/alibaba-cloud/README.md index 98f914a693..7b45551f5c 100644 --- a/deploy/alibaba-cloud/README.md +++ b/deploy/alibaba-cloud/README.md @@ -1,5 +1,5 @@ +# OpenAI-Compatible Frontend for Triton Inference Server (Beta) + +> [!NOTE] +> The OpenAI-Compatible API is currently in BETA. Its features and functionality +> are subject to change as we collect feedback. We're excited to hear any thoughts +> you have and what features you'd like to see! + +## Pre-requisites + +1. Docker + NVIDIA Container Runtime +2. A correctly configured `HF_TOKEN` for access to HuggingFace models. + - The current examples and testing primarily use the + [`meta-llama/Meta-Llama-3.1-8B-Instruct`](https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct) + model, but you can manually bring your own models and adjust accordingly. + +## VLLM + +1. Launch the container and install dependencies: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 +``` + +2. Launch the OpenAI-compatible Triton Inference Server: +```bash +cd /opt/tritonserver/python/openai + +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository tests/vllm_models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +
+Example output + +``` +... ++-----------------------+---------+--------+ +| Model | Version | Status | ++-----------------------+---------+--------+ +| llama-3.1-8b-instruct | 1 | READY | <- Correct Model Loaded in Triton ++-----------------------+---------+--------+ +... +Found model: name='llama-3.1-8b-instruct', backend='vllm' +[WARNING] Adding CORS for the following origins: ['http://localhost'] +INFO: Started server process [126] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully +``` + +
+ +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-6930b296-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": + { + "content": "This is only a test.", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1727679085, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +4. Send a `/v1/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="llama-3.1-8b-instruct" +curl -s http://localhost:9000/v1/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "prompt": "Machine learning is" +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-d51df75c-7ef8-11ef-bdd1-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "logprobs": null, + "text": " a field of computer science that focuses on developing algorithms that allow computers to learn from" + } + ], + "created": 1727679266, + "model": "llama-3.1-8b-instruct", + "system_fingerprint": null, + "object": "text_completion", + "usage": null +} +``` + +
+ +5. Benchmark with `genai-perf`: +- To install genai-perf in this container, see the instructions [here](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) +- Or try using genai-perf from the [SDK container](https://github.com/triton-inference-server/perf_analyzer/tree/main/genai-perf#install-perf-analyzer-ubuntu-python-38) + +```bash +MODEL="llama-3.1-8b-instruct" +TOKENIZER="meta-llama/Meta-Llama-3.1-8B-Instruct" +genai-perf profile \ + --model ${MODEL} \ + --tokenizer ${TOKENIZER} \ + --service-kind openai \ + --endpoint-type chat \ + --url localhost:9000 \ + --streaming +``` + +
+Example output + +``` +2024-10-14 22:43 [INFO] genai_perf.parser:82 - Profiling these models: llama-3.1-8b-instruct +2024-10-14 22:43 [INFO] genai_perf.wrapper:163 - Running Perf Analyzer : 'perf_analyzer -m llama-3.1-8b-instruct --async --input-data artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/inputs.json -i http --concurrency-range 1 --endpoint v1/chat/completions --service-kind openai -u localhost:9000 --measurement-interval 10000 --stability-percentage 999 --profile-export-file artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export.json' + NVIDIA GenAI-Perf | LLM Metrics +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ +┃ Statistic ┃ avg ┃ min ┃ max ┃ p99 ┃ p90 ┃ p75 ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ +│ Time to first token (ms) │ 71.66 │ 64.32 │ 86.52 │ 76.13 │ 74.92 │ 73.26 │ +│ Inter token latency (ms) │ 18.47 │ 18.25 │ 18.72 │ 18.67 │ 18.61 │ 18.53 │ +│ Request latency (ms) │ 348.00 │ 274.60 │ 362.27 │ 355.41 │ 352.29 │ 350.66 │ +│ Output sequence length │ 15.96 │ 12.00 │ 16.00 │ 16.00 │ 16.00 │ 16.00 │ +│ Input sequence length │ 549.66 │ 548.00 │ 551.00 │ 550.00 │ 550.00 │ 550.00 │ +│ Output token throughput (per sec) │ 45.84 │ N/A │ N/A │ N/A │ N/A │ N/A │ +│ Request throughput (per sec) │ 2.87 │ N/A │ N/A │ N/A │ N/A │ N/A │ +└───────────────────────────────────┴────────┴────────┴────────┴────────┴────────┴────────┘ +2024-10-14 22:44 [INFO] genai_perf.export_data.json_exporter:62 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.json +2024-10-14 22:44 [INFO] genai_perf.export_data.csv_exporter:71 - Generating artifacts/llama-3.1-8b-instruct-openai-chat-concurrency1/profile_export_genai_perf.csv +``` + +
+ +6. Use the OpenAI python client directly: +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:9000/v1", + api_key="EMPTY", +) + +model = "llama-3.1-8b-instruct" +completion = client.chat.completions.create( + model=model, + messages=[ + { + "role": "system", + "content": "You are a helpful assistant.", + }, + {"role": "user", "content": "What are LLMs?"}, + ], + max_tokens=256, +) + +print(completion.choices[0].message.content) +``` + +7. Run tests (NOTE: The server should not be running, the tests will handle starting/stopping the server as necessary): +```bash +cd /opt/tritonserver/python/openai/ +pip install -r requirements-test.txt + +pytest -v tests/ +``` + +## TensorRT-LLM + +0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: + - [Triton CLI](https://github.com/triton-inference-server/triton_cli/) + - [TRT-LLM Backend Quickstart](https://github.com/triton-inference-server/tensorrtllm_backend?tab=readme-ov-file#quick-start) + +1. Launch the container: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + -e TRTLLM_ORCHESTRATOR=1 \ + nvcr.io/nvidia/tritonserver:24.11-trtllm-python-py3 +``` + +2. Install dependencies inside the container: +```bash +# Install python bindings for tritonserver and tritonfrontend +pip install /opt/tritonserver/python/triton*.whl + +# Install application requirements +git clone https://github.com/triton-inference-server/server.git +cd server/python/openai/ +pip install -r requirements.txt +``` + +2. Launch the OpenAI server: +```bash +# NOTE: Adjust the --tokenizer based on the model being used +python3 openai_frontend/main.py --model-repository path/to/models --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct +``` + +3. Send a `/v1/chat/completions` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +# MODEL should be the client-facing model name in your model repository for a pipeline like TRT-LLM. +# For example, this could also be "ensemble", or something like "gpt2" if generated from Triton CLI +MODEL="tensorrt_llm_bls" +curl -s http://localhost:9000/v1/chat/completions -H 'Content-Type: application/json' -d '{ + "model": "'${MODEL}'", + "messages": [{"role": "user", "content": "Say this is a test!"}] +}' | jq +``` + +
+Example output + +```json +{ + "id": "cmpl-704c758c-8a84-11ef-b106-107c6149ca79", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "It looks like you're testing the system!", + "tool_calls": null, + "role": "assistant", + "function_call": null + }, + "logprobs": null + } + ], + "created": 1728948689, + "model": "llama-3-8b-instruct", + "system_fingerprint": null, + "object": "chat.completion", + "usage": null +} +``` + +
+ +The other examples should be the same as vLLM, except that you should set `MODEL="tensorrt_llm_bls"` or `MODEL="ensemble"`, +everywhere applicable as seen in the example request above. + +## KServe Frontends + +To support serving requests through both the OpenAI-Compatible and +KServe Predict v2 frontends to the same running Triton Inference Server, +the `tritonfrontend` python bindings are included for optional use in this +application as well. + +You can opt-in to including these additional frontends, assuming `tritonfrontend` +is installed, with `--enable-kserve-frontends` like below: + +``` +python3 openai_frontend/main.py \ + --model-repository tests/vllm_models \ + --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \ + --enable-kserve-frontends +``` + +See `python3 openai_frontend/main.py --help` for more information on the +available arguments and default values. + +For more information on the `tritonfrontend` python bindings, see the docs +[here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md). + +## Model Parallelism Support + +- [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend)) + - ex: Configure `tensor_parallel_size: 2` in the + [model.json](https://github.com/triton-inference-server/vllm_backend/blob/main/samples/model_repository/vllm_model/1/model.json) +- [x] TensorRT-LLM ([Orchestrator Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#orchestrator-mode)) + - Set the following environment variable: `export TRTLLM_ORCHESTRATOR=1` +- [ ] TensorRT-LLM ([Leader Mode](https://github.com/triton-inference-server/tensorrtllm_backend/blob/main/README.md#leader-mode)) + - Not currently supported diff --git a/docs/contents.rst b/docs/contents.rst index 555c433d85..dfff933e31 100644 --- a/docs/contents.rst +++ b/docs/contents.rst @@ -50,10 +50,11 @@ .. toctree:: :hidden: - :caption: AI Agents + :caption: LLM Features - Constrained Decoding <../tutorials/AI_Agents_Guide/Constrained_Decoding/README.md> - Function Calling <../tutorials/AI_Agents_Guide/Function_Calling/README.md> + Constrained Decoding <../tutorials/Feature_Guide/Constrained_Decoding/README.md> + Function Calling <../tutorials/Feature_Guide/Function_Calling/README.md> + llm_features/speculative_decoding_by_backend_type .. toctree:: :hidden: diff --git a/docs/examples/fetch_models.sh b/docs/examples/fetch_models.sh index f5aaed85aa..c0c97c5b85 100755 --- a/docs/examples/fetch_models.sh +++ b/docs/examples/fetch_models.sh @@ -1,5 +1,6 @@ #!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. + +# Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,12 +28,19 @@ set -ex -# TensorFlow inception -mkdir -p model_repository/inception_graphdef/1 +# Convert Tensorflow inception V3 module to ONNX +# Pre-requisite: Python3, venv, and Pip3 are installed on the system +mkdir -p model_repository/inception_onnx/1 wget -O /tmp/inception_v3_2016_08_28_frozen.pb.tar.gz \ https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz (cd /tmp && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) -mv /tmp/inception_v3_2016_08_28_frozen.pb model_repository/inception_graphdef/1/model.graphdef +python3 -m venv tf2onnx +source ./tf2onnx/bin/activate +pip3 install "numpy<2" tensorflow tf2onnx +python3 -m tf2onnx.convert --graphdef /tmp/inception_v3_2016_08_28_frozen.pb --output inception_v3_onnx.model.onnx --inputs input:0 --outputs InceptionV3/Predictions/Softmax:0 +deactivate +mv inception_v3_onnx.model.onnx model_repository/inception_onnx/1/model.onnx + # ONNX densenet mkdir -p model_repository/densenet_onnx/1 diff --git a/docs/examples/model_repository/inception_graphdef/config.pbtxt b/docs/examples/model_repository/inception_graphdef/config.pbtxt deleted file mode 100644 index 1636d56f77..0000000000 --- a/docs/examples/model_repository/inception_graphdef/config.pbtxt +++ /dev/null @@ -1,19 +0,0 @@ -name: "inception_graphdef" -platform: "tensorflow_graphdef" -max_batch_size: 128 -input [ - { - name: "input" - data_type: TYPE_FP32 - format: FORMAT_NHWC - dims: [ 299, 299, 3 ] - } -] -output [ - { - name: "InceptionV3/Predictions/Softmax" - data_type: TYPE_FP32 - dims: [ 1001 ] - label_filename: "inception_labels.txt" - } -] diff --git a/docs/examples/model_repository/inception_onnx/config.pbtxt b/docs/examples/model_repository/inception_onnx/config.pbtxt new file mode 100644 index 0000000000..c4f11b41ef --- /dev/null +++ b/docs/examples/model_repository/inception_onnx/config.pbtxt @@ -0,0 +1,44 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +name: "inception_onnx" +platform: "onnxruntime_onnx" +max_batch_size: 128 +input [ + { + name: "input" + data_type: TYPE_FP32 + format: FORMAT_NHWC + dims: [ 299, 299, 3 ] + } +] +output [ + { + name: "InceptionV3/Predictions/Softmax" + data_type: TYPE_FP32 + dims: [ 1001 ] + label_filename: "inception_labels.txt" + } +] diff --git a/docs/examples/model_repository/inception_graphdef/inception_labels.txt b/docs/examples/model_repository/inception_onnx/inception_labels.txt similarity index 100% rename from docs/examples/model_repository/inception_graphdef/inception_labels.txt rename to docs/examples/model_repository/inception_onnx/inception_labels.txt diff --git a/docs/examples/model_repository/simple/1/model.graphdef b/docs/examples/model_repository/simple/1/model.graphdef deleted file mode 100644 index d7409a4429..0000000000 --- a/docs/examples/model_repository/simple/1/model.graphdef +++ /dev/null @@ -1,21 +0,0 @@ - -@ -INPUT0 Placeholder* -shape: * -dtype0 -@ -INPUT1 Placeholder* -dtype0* -shape:  -2 -ADDAddINPUT0INPUT1" /device:CPU:0* -T0 -2 -SUBSubINPUT0INPUT1" /device:CPU:0* -T0 -! -OUTPUT0IdentityADD* -T0 -! -OUTPUT1IdentitySUB* -T0" \ No newline at end of file diff --git a/docs/examples/model_repository/simple/1/model.onnx b/docs/examples/model_repository/simple/1/model.onnx new file mode 100755 index 0000000000..21e7178367 Binary files /dev/null and b/docs/examples/model_repository/simple/1/model.onnx differ diff --git a/docs/examples/model_repository/simple/config.pbtxt b/docs/examples/model_repository/simple/config.pbtxt index b33ac77a51..1586e726ea 100644 --- a/docs/examples/model_repository/simple/config.pbtxt +++ b/docs/examples/model_repository/simple/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 input [ { diff --git a/docs/examples/model_repository/simple_dyna_sequence/1/model.graphdef b/docs/examples/model_repository/simple_dyna_sequence/1/model.graphdef deleted file mode 100755 index 7dbacf70b4..0000000000 Binary files a/docs/examples/model_repository/simple_dyna_sequence/1/model.graphdef and /dev/null differ diff --git a/docs/examples/model_repository/simple_dyna_sequence/1/model.onnx b/docs/examples/model_repository/simple_dyna_sequence/1/model.onnx new file mode 100755 index 0000000000..dedd2ac284 Binary files /dev/null and b/docs/examples/model_repository/simple_dyna_sequence/1/model.onnx differ diff --git a/docs/examples/model_repository/simple_dyna_sequence/config.pbtxt b/docs/examples/model_repository/simple_dyna_sequence/config.pbtxt index 47889f1f7c..1f11b73bac 100644 --- a/docs/examples/model_repository/simple_dyna_sequence/config.pbtxt +++ b/docs/examples/model_repository/simple_dyna_sequence/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple_dyna_sequence" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 sequence_batching { max_sequence_idle_microseconds: 10000000 diff --git a/qa/L0_model_config/noautofill_test.py b/docs/examples/model_repository/simple_identity/1/model.py old mode 100755 new mode 100644 similarity index 54% rename from qa/L0_model_config/noautofill_test.py rename to docs/examples/model_repository/simple_identity/1/model.py index d89e306eb8..906c173892 --- a/qa/L0_model_config/noautofill_test.py +++ b/docs/examples/model_repository/simple_identity/1/model.py @@ -1,5 +1,4 @@ -#!/usr/bin/python -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,38 +24,23 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import sys +import json -sys.path.append("../common") +import triton_python_backend_utils as pb_utils -import unittest -import test_util as tu -import tritonclient.http as httpclient -from tritonclient.utils import InferenceServerException +class TritonPythonModel: + """This model always returns the input that it has received.""" + def initialize(self, args): + self.model_config = json.loads(args["model_config"]) -class NoAutoFillTest(tu.TestResultCollector): - def setUp(self): - self._model_name = "noautofill_noconfig" - self._triton_client = httpclient.InferenceServerClient("localhost:8000") + def execute(self, requests): + """This function is called on inference request.""" - def tearDown(self): - self._triton_client.unload_model(self._model_name) - - def test_load_no_autofill_model_with_config(self): - config = '{"max_batch_size":"16"}' - self._triton_client.load_model(self._model_name, config=config) - - # Check if the model config is correct - model_config = self._triton_client.get_model_config(self._model_name) - self.assertEqual(model_config["max_batch_size"], 16) - - def test_load_no_autofill_model_with_no_config(self): - with self.assertRaises(InferenceServerException) as ex: - self._triton_client.load_model(self._model_name) - self.assertIn("model configuration is not provided", str(ex.exception)) - - -if __name__ == "__main__": - unittest.main() + responses = [] + for request in requests: + in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor_0 = pb_utils.Tensor("OUTPUT0", in_0.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor_0])) + return responses diff --git a/docs/examples/model_repository/simple_identity/1/model.savedmodel/saved_model.pb b/docs/examples/model_repository/simple_identity/1/model.savedmodel/saved_model.pb deleted file mode 100755 index 63f78fecb4..0000000000 Binary files a/docs/examples/model_repository/simple_identity/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/docs/examples/model_repository/simple_identity/config.pbtxt b/docs/examples/model_repository/simple_identity/config.pbtxt index fa7baee9c6..eccf40e326 100644 --- a/docs/examples/model_repository/simple_identity/config.pbtxt +++ b/docs/examples/model_repository/simple_identity/config.pbtxt @@ -1,6 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple_identity" -platform: "tensorflow_savedmodel" +backend: "python" max_batch_size: 8 input [ diff --git a/docs/examples/model_repository/simple_int8/1/model.graphdef b/docs/examples/model_repository/simple_int8/1/model.graphdef deleted file mode 100755 index 65cbc0dcf4..0000000000 --- a/docs/examples/model_repository/simple_int8/1/model.graphdef +++ /dev/null @@ -1,21 +0,0 @@ - -@ -INPUT0 Placeholder* -dtype0* -shape:  -@ -INPUT1 Placeholder* -dtype0* -shape:  -# -ADDAddINPUT0INPUT1* -T0 -# -SUBSubINPUT0INPUT1* -T0 -! -OUTPUT0IdentityADD* -T0 -! -OUTPUT1IdentitySUB* -T0" \ No newline at end of file diff --git a/docs/examples/model_repository/simple_int8/1/model.onnx b/docs/examples/model_repository/simple_int8/1/model.onnx new file mode 100755 index 0000000000..e38fc52c80 Binary files /dev/null and b/docs/examples/model_repository/simple_int8/1/model.onnx differ diff --git a/docs/examples/model_repository/simple_int8/config.pbtxt b/docs/examples/model_repository/simple_int8/config.pbtxt index 47e3324456..ba307d76a0 100644 --- a/docs/examples/model_repository/simple_int8/config.pbtxt +++ b/docs/examples/model_repository/simple_int8/config.pbtxt @@ -1,5 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + name: "simple_int8" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 input [ { diff --git a/docs/examples/model_repository/simple_sequence/1/model.graphdef b/docs/examples/model_repository/simple_sequence/1/model.graphdef deleted file mode 100755 index d4c4bd6031..0000000000 Binary files a/docs/examples/model_repository/simple_sequence/1/model.graphdef and /dev/null differ diff --git a/docs/examples/model_repository/simple_sequence/1/model.onnx b/docs/examples/model_repository/simple_sequence/1/model.onnx new file mode 100755 index 0000000000..4f7283313e Binary files /dev/null and b/docs/examples/model_repository/simple_sequence/1/model.onnx differ diff --git a/docs/examples/model_repository/simple_sequence/config.pbtxt b/docs/examples/model_repository/simple_sequence/config.pbtxt index 1dd5c0da7c..d63f4a60ac 100644 --- a/docs/examples/model_repository/simple_sequence/config.pbtxt +++ b/docs/examples/model_repository/simple_sequence/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple_sequence" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 sequence_batching { control_input [ diff --git a/docs/examples/model_repository/simple_string/1/model.graphdef b/docs/examples/model_repository/simple_string/1/model.graphdef deleted file mode 100644 index d2d3db9180..0000000000 Binary files a/docs/examples/model_repository/simple_string/1/model.graphdef and /dev/null differ diff --git a/docs/examples/model_repository/simple_string/1/model.onnx b/docs/examples/model_repository/simple_string/1/model.onnx new file mode 100755 index 0000000000..490b70392a Binary files /dev/null and b/docs/examples/model_repository/simple_string/1/model.onnx differ diff --git a/docs/examples/model_repository/simple_string/config.pbtxt b/docs/examples/model_repository/simple_string/config.pbtxt index b01cd039b0..a9c70aae62 100644 --- a/docs/examples/model_repository/simple_string/config.pbtxt +++ b/docs/examples/model_repository/simple_string/config.pbtxt @@ -1,6 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple_string" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 input [ { diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index be6e1da66f..68a154f4bf 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -37,6 +37,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.03 | nvcr.io/nvidia/tritonserver:25.03-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0%2B7c8ec84dab.nv25.3 | 10.9.0.34 | 0.18.0 | 12.8.1.012 | 570.124.06 | 28G | | 25.02 | nvcr.io/nvidia/tritonserver:25.02-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Becf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0.post1 | 12.8.0.038 | 570.86.10 | 28G | | 25.01 | nvcr.io/nvidia/tritonserver:25.01-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Becf3bae40a.nv25.1 | 10.8.0.43 | 0.17.0 | 12.8.0.038 | 570.86.10 | 30G | | 24.12 | nvcr.io/nvidia/tritonserver:24.12-trtllm-python-py3 | Python 3.12.3 | 2.6.0a0%2Bdf5bbc09d1.nv24.11 | 10.7.0 | 0.16.0 | 12.6.3 | 560.35.05 | 22G | @@ -53,6 +54,7 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | +| 25.03 | nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 | Python 3.12.3 | 0.7.3+04de634a.nv25.3.cu128 | 12.8.1.012 | 570.124.06 | 22G | | 25.02 | nvcr.io/nvidia/tritonserver:25.02-vllm-python-py3 | Python 3.12.3 | 0.7.0+5e800e3d.nv25.2.cu128 | 12.8.0.038 | 570.86.10 | 22G | | 25.01 | nvcr.io/nvidia/tritonserver:25.01-vllm-python-py3 | Python 3.12.3 | 0.6.3.post1 | 12.8.0.038 | 570.86.10 | 23G | | 24.12 | nvcr.io/nvidia/tritonserver:24.12-vllm-python-py3 | Python 3.12.3 | 0.5.5 | 12.6.3.004 | 560.35.05 | 20G | @@ -69,6 +71,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 25.03 | 1.21.0 | | 25.02 | 1.20.1 | | 25.01 | 1.20.1 | | 24.12 | 1.20.1 | diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index b2d5edadf2..d0516e23c3 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.02](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-02.html#rel-25-02) +# [Triton Inference Server Release 25.03](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-03.html#rel-25-03) -The Triton Inference Server container image, release 25.02, is available +The Triton Inference Server container image, release 25.03, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/docs/llm_features/speculative_decoding.rst b/docs/llm_features/speculative_decoding.rst new file mode 100644 index 0000000000..debbcf52ae --- /dev/null +++ b/docs/llm_features/speculative_decoding.rst @@ -0,0 +1,54 @@ +.. +.. Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +.. raw:: html + + +About Speculative Decoding +========================= +Speculative Decoding (also referred to as Speculative Sampling) is a set of techniques designed +to allow generation of more than one token per forward pass iteration. This can lead to a reduction +in the average per-token latency in situations where the GPU is underutilized due to small batch sizes. + +Speculative decoding involves predicting a sequence of future tokens, referred to as draft tokens, +using a method that is substantially more efficient than repeatedly executing the target Large Language +Model (LLM). These draft tokens are then collectively validated by processing them through the target LLM +in a single forward pass. The underlying assumptions are twofold: + +1. processing multiple draft tokens concurrently will be as rapid as processing a single token +2. multiple draft tokens will be validated successfully over the course of the full generation + +If the first assumption holds true, the latency of speculative decoding will no worse than the standard +approach. If the second holds, output token generation advances by statistically more than one token per +forward pass. The combination of both these allows speculative decoding to result in reduced latency. + +Performance Improvements +======================== +It's important to note that the effectiveness of speculative decoding techniques is highly dependent +on the specific task at hand. For instance, forecasting subsequent tokens in a code-completion scenario +may prove simpler than generating a summary for an article. `Spec-Bench `__ +shows the performance of different speculative decoding approaches on different tasks. \ No newline at end of file diff --git a/docs/llm_features/speculative_decoding_by_backend_type.rst b/docs/llm_features/speculative_decoding_by_backend_type.rst new file mode 100644 index 0000000000..a61f625626 --- /dev/null +++ b/docs/llm_features/speculative_decoding_by_backend_type.rst @@ -0,0 +1,39 @@ +.. +.. Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. +.. Redistribution and use in source and binary forms, with or without +.. modification, are permitted provided that the following conditions +.. are met: +.. * Redistributions of source code must retain the above copyright +.. notice, this list of conditions and the following disclaimer. +.. * Redistributions in binary form must reproduce the above copyright +.. notice, this list of conditions and the following disclaimer in the +.. documentation and/or other materials provided with the distribution. +.. * Neither the name of NVIDIA CORPORATION nor the names of its +.. contributors may be used to endorse or promote products derived +.. from this software without specific prior written permission. +.. +.. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +.. EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +.. PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +.. CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +.. EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +.. PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +.. PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +.. OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +.. (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +.. OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#### +Speculative Decoding +#### + +.. include:: speculative_decoding.rst + +.. toctree:: + :maxdepth: 1 + :hidden: + + TRT-LLM <../tutorials/Feature_Guide/Speculative_Decoding/TRT-LLM/README.md> + vLLM <../tutorials/Feature_Guide/Speculative_Decoding/vLLM/README.md> \ No newline at end of file diff --git a/python/openai/README.md b/python/openai/README.md index c5a3cb4d0a..53a9f461f4 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.02-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.03-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: diff --git a/qa/L0_backend_config/test.sh b/qa/L0_backend_config/test.sh index dad586883b..16625746b3 100755 --- a/qa/L0_backend_config/test.sh +++ b/qa/L0_backend_config/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -57,7 +57,7 @@ fi rm -rf ./models/ mkdir -p ./models/no_config -cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/savedmodel_float32_float32_float32/1 ./models/no_config/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/onnx_float32_float32_float32/1 ./models/no_config/ SERVER=/opt/tritonserver/bin/tritonserver @@ -73,14 +73,12 @@ COMMON_ARGS="--model-repository=`pwd`/models --strict-model-config=false --log-v NEGATIVE_PARSE_ARGS=("--backend-config=,default-max-batch-size=3 $COMMON_ARGS" \ "--backend-config=default-max-batch-size= $COMMON_ARGS" \ "--backend-config=default-max-batch-size $COMMON_ARGS" \ - "--backend-config=tensorflow,default-max-batch-size= $COMMON_ARGS" \ - "--backend-config=tensorflow,default-max-batch-size $COMMON_ARGS" \ ) POSITIVE_DEFAULT_ARGS=$COMMON_ARGS -POSITIVE_TEST_ARGS=("--backend-config=tensorflow,default-max-batch-size=5 $COMMON_ARGS" \ +POSITIVE_TEST_ARGS=("--backend-config=default-max-batch-size=5 $COMMON_ARGS" \ "--backend-config=default-max-batch-size=6 $COMMON_ARGS" \ - "--backend-config=default-max-batch-size=7 --backend-config=tensorflow,default-max-batch-size=8 $COMMON_ARGS" \ + "--backend-config=default-max-batch-size=7 --backend-config=default-max-batch-size=8 $COMMON_ARGS" \ ) # These integers correspond to the expected default-max-batch-size which gets set @@ -185,74 +183,6 @@ function save_model_config() { fi } -# Tensorflow 1: Batching ON -rm -rf ./models/ -mkdir -p ./models/no_config -cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/savedmodel_float32_float32_float32/1 ./models/no_config/ - -SERVER_ARGS="--backend-config=tensorflow,default-max-batch-size=5 $COMMON_ARGS" -SERVER_LOG=$SERVER_LOG_BASE.backend_config_tensorflow_batch_5.log -run_server - -TRIAL=tensorflow_batching_on -if [ "$SERVER_PID" == "0" ]; then - echo -e "*** FAILED: Server failed to start $SERVER\n" - RET=1 -else - save_model_config - - # Assert the max-batch-size is the command line value - MAX_BATCH_LOG_LINE=$(grep -a "\"max_batch_size\":5" $TRIAL.out) - if [ "$MAX_BATCH_LOG_LINE" == "" ]; then - cat $TRIAL.out - echo "*** FAILED: Expected max batch size to be 5 but found: $MAX_BATCH_LOG_LINE\n" - RET=1 - fi - - # Assert we are also turning on the dynamic_batcher - DYNAMIC_BATCHING_LOG_LINE=$(grep -a "Starting dynamic-batcher thread" $SERVER_LOG) - if [ "$DYNAMIC_BATCHING_LOG_LINE" == "" ]; then - echo "*** FAILED: Expected dynamic batching to be set in model config but was not found\n" - RET=1 - fi - - kill $SERVER_PID - wait $SERVER_PID - -fi - -# Tensorflow 1: Batching OFF -SERVER_ARGS="--backend-config=tensorflow,default-max-batch-size=0 $COMMON_ARGS" -SERVER_LOG=$SERVER_LOG_BASE.backend_config_tensorflow_batch_0.log -run_server - -TRIAL=tensorflow_batching_off -if [ "$SERVER_PID" == "0" ]; then - echo -e "*** FAILED: Server failed to start $SERVER\n" - RET=1 - -else - save_model_config - - # Assert the max-batch-size is 0 in the case batching is supported - # in the model but not in the config. - MAX_BATCH_LOG_LINE=$(grep -a "\"max_batch_size\":0" $TRIAL.out) - if [ "$MAX_BATCH_LOG_LINE" == "" ]; then - echo "*** FAILED: Expected max batch size to be 0 but found: $MAX_BATCH_LOG_LINE\n" - RET=1 - fi - - # Assert batching disabled - if [ "$(grep -a -E '\"dynamic_batching\": \{}' $SERVER_LOG)" != "" ]; then - echo "*** FAILED: Found dynamic batching enabled in configuration when none expected.\n" - RET=1 - fi - - kill $SERVER_PID - wait $SERVER_PID - -fi - # Onnxruntime: Batching ON rm -rf ./models/ mkdir -p ./models/no_config diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh index 0279d7984c..4d9de30a56 100755 --- a/qa/L0_backend_python/env/test.sh +++ b/qa/L0_backend_python/env/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,77 +41,6 @@ rm -rf *.tar.gz install_build_deps install_conda -# Tensorflow 2.1.0 only works with Python 3.4 - 3.7. Successful execution of -# the Python model indicates that the environment has been setup correctly. -# Create a model with python 3.7 version -export PY_VERSION="3.7" -create_conda_env "3.7" "python-3-7" -conda install numpy=1.20.1 -y -conda install tensorflow=2.1.0 -y -conda install -c conda-forge libstdcxx-ng=14 -y - -PY37_VERSION_STRING="Python version is 3.7, NumPy version is 1.20.1, and Tensorflow version is 2.1.0" -create_python_backend_stub -conda-pack -o python3.7.tar.gz -path_to_conda_pack=`pwd`/python3.7.tar.gz -mkdir -p models/python_3_7/1/ -cp ../../python_models/python_version/config.pbtxt ./models/python_3_7 -(cd models/python_3_7 && \ - sed -i "s/^name:.*/name: \"python_3_7\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) -cp ../../python_models/python_version/model.py ./models/python_3_7/1/ -cp python_backend/builddir/triton_python_backend_stub ./models/python_3_7 -conda deactivate - -# Use python-3-7 without conda pack -# Create a model with python 3.7 version and numpy 1.20.3 to distinguish from -# previous test. -# Tensorflow 2.1.0 only works with Python 3.4 - 3.7. Successful execution of -# the Python model indicates that the environment has been setup correctly. -export PY_VERSION="3.7.1" -path_to_conda_pack="$PWD/python-3-7-1" -create_conda_env_with_specified_path "3.7" $path_to_conda_pack -conda install numpy=1.20.3 -y -conda install tensorflow=2.1.0 -y -conda install -c conda-forge libstdcxx-ng=14 -y - -PY37_1_VERSION_STRING="Python version is 3.7, NumPy version is 1.20.3, and Tensorflow version is 2.1.0" -create_python_backend_stub -mkdir -p models/python_3_7_1/1/ -cp ../../python_models/python_version/config.pbtxt ./models/python_3_7_1 -(cd models/python_3_7_1 && \ - sed -i "s/^name:.*/name: \"python_3_7_1\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) -cp ../../python_models/python_version/model.py ./models/python_3_7_1/1/ -# Copy activate script to folder -cp $path_to_conda_pack/lib/python3.7/site-packages/conda_pack/scripts/posix/activate $path_to_conda_pack/bin/. -cp python_backend/builddir/triton_python_backend_stub ./models/python_3_7_1 -conda deactivate - -# Create a model with python 3.6 version -# Tensorflow 2.1.0 only works with Python 3.4 - 3.7. Successful execution of -# the Python model indicates that the environment has been setup correctly. -export PY_VERSION="3.6" -create_conda_env "3.6" "python-3-6" -conda install -c conda-forge libstdcxx-ng=14 -y -conda install numpy=1.18.1 -y -conda install tensorflow=2.1.0 -y -PY36_VERSION_STRING="Python version is 3.6, NumPy version is 1.18.1, and Tensorflow version is 2.1.0" -conda-pack -o python3.6.tar.gz - -# Test relative execution env path -path_to_conda_pack='$$TRITON_MODEL_DIRECTORY/python_3_6_environment.tar.gz' -create_python_backend_stub -mkdir -p models/python_3_6/1/ -cp ../../python_models/python_version/config.pbtxt ./models/python_3_6 -cp python3.6.tar.gz models/python_3_6/python_3_6_environment.tar.gz -(cd models/python_3_6 && \ - sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt) -cp ../../python_models/python_version/model.py ./models/python_3_6/1/ -cp python_backend/builddir/triton_python_backend_stub ./models/python_3_6 -conda deactivate - # Test conda env without custom Python backend stub This environment should # always use the default Python version shipped in the container. For Ubuntu # 24.04 it is Python 3.12, for Ubuntu 22.04 is Python 3.10 and for Ubuntu 20.04 @@ -119,13 +48,13 @@ conda deactivate path_to_conda_pack='$$TRITON_MODEL_DIRECTORY/python_3_12_environment.tar.gz' create_conda_env "3.12" "python-3-12" conda install -c conda-forge libstdcxx-ng=14 -y -TF_VERSION="2.16.2" +TORCH_VERSION="2.6.0" conda install numpy=1.26.4 -y if [ $TRITON_RHEL -eq 1 ]; then - TF_VERSION="2.17.0" + TORCH_VERISON="2.17.0" fi -conda install tensorflow=${TF_VERSION} -y -PY312_VERSION_STRING="Python version is 3.12, NumPy version is 1.26.4, and Tensorflow version is ${TF_VERSION}" +conda install torch=${TORCH_VERSION} -y +PY312_VERSION_STRING="Python version is 3.12, NumPy version is 1.26.4, and PyTorch version is ${TORCH_VERISON}" conda pack -o python3.12.tar.gz mkdir -p models/python_3_12/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_12 @@ -147,7 +76,7 @@ fi kill_server set +e -for EXPECTED_VERSION_STRING in "$PY36_VERSION_STRING" "$PY37_VERSION_STRING" "$PY37_1_VERSION_STRING" "$PY312_VERSION_STRING"; do +for EXPECTED_VERSION_STRING in "$PY312_VERSION_STRING"; do grep "$EXPECTED_VERSION_STRING" $SERVER_LOG if [ $? -ne 0 ]; then cat $SERVER_LOG diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh index c84c7f5ec8..c3d0a0aed7 100755 --- a/qa/L0_backend_python/setup_python_enviroment.sh +++ b/qa/L0_backend_python/setup_python_enviroment.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,84 +50,16 @@ install_conda # Test other python versions conda update -n base -c defaults conda -y -# Create a model with python 3.8 version -# Successful execution of the Python model indicates that the environment has -# been setup correctly. -if [ ${PYTHON_ENV_VERSION} = "8" ]; then - create_conda_env "3.8" "python-3-8" - conda install -c conda-forge libstdcxx-ng=14 -y - conda install numpy=1.23.4 -y - conda install tensorflow=2.10.0 -y - EXPECTED_VERSION_STRING="Python version is 3.8, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" - create_python_backend_stub - conda-pack -o python3.8.tar.gz - path_to_conda_pack="$PWD/python-3-8" - mkdir -p $path_to_conda_pack - tar -xzf python3.8.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_8/1/ - cp ../python_models/python_version/config.pbtxt ./models/python_3_8 - (cd models/python_3_8 && \ - sed -i "s/^name:.*/name: \"python_3_8\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../python_models/python_version/model.py ./models/python_3_8/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_8 -fi - -# Create a model with python 3.9 version -# Successful execution of the Python model indicates that the environment has -# been setup correctly. -if [ ${PYTHON_ENV_VERSION} = "9" ]; then - create_conda_env "3.9" "python-3-9" - conda install -c conda-forge libstdcxx-ng=14 -y - conda install numpy=1.23.4 -y - conda install tensorflow=2.10.0 -y - EXPECTED_VERSION_STRING="Python version is 3.9, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" - create_python_backend_stub - conda-pack -o python3.9.tar.gz - path_to_conda_pack="$PWD/python-3-9" - mkdir -p $path_to_conda_pack - tar -xzf python3.9.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_9/1/ - cp ../python_models/python_version/config.pbtxt ./models/python_3_9 - (cd models/python_3_9 && \ - sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../python_models/python_version/model.py ./models/python_3_9/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9 -fi - -# Create a model with python 3.10 version -# Successful execution of the Python model indicates that the environment has -# been setup correctly. -if [ ${PYTHON_ENV_VERSION} = "10" ]; then - create_conda_env "3.10" "python-3-10" - conda install -c conda-forge libstdcxx-ng=14 -y - conda install tensorflow=2.10.0 -y - conda install numpy=1.23.4 -y - EXPECTED_VERSION_STRING="Python version is 3.10, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" - create_python_backend_stub - conda-pack -o python3.10.tar.gz - path_to_conda_pack="$PWD/python-3-10" - mkdir -p $path_to_conda_pack - tar -xzf python3.10.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_10/1/ - cp ../python_models/python_version/config.pbtxt ./models/python_3_10 - (cd models/python_3_10 && \ - sed -i "s/^name:.*/name: \"python_3_10\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../python_models/python_version/model.py ./models/python_3_10/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_10 -fi # Create a model with python 3.11 version # Successful execution of the Python model indicates that the environment has # been setup correctly. if [ ${PYTHON_ENV_VERSION} = "11" ]; then create_conda_env "3.11" "python-3-11" - conda install tensorflow=2.12.0 -y + conda install torch=2.6.0 -y conda install -c conda-forge libstdcxx-ng=14 -y conda install numpy=1.23.5 -y - EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and Tensorflow version is 2.12.0" + EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and PyTorch version is 2.6.0" create_python_backend_stub conda-pack -o python3.11.tar.gz path_to_conda_pack="$PWD/python-3-11" diff --git a/qa/L0_batch_custom/batch_custom_test.py b/qa/L0_batch_custom/batch_custom_test.py index 6cd6346ad3..1bd5c0d7f2 100755 --- a/qa/L0_batch_custom/batch_custom_test.py +++ b/qa/L0_batch_custom/batch_custom_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -88,9 +88,7 @@ def check_response( start_ms = int(round(time.time() * 1000)) if ( - trial == "savedmodel" - or trial == "graphdef" - or trial == "libtorch" + trial == "libtorch" or trial == "onnx" or trial == "plan" or trial == "python" diff --git a/qa/L0_batch_input/test.sh b/qa/L0_batch_input/test.sh index e780516ec4..6c2b0c0d69 100755 --- a/qa/L0_batch_input/test.sh +++ b/qa/L0_batch_input/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ SERVER_LOG="./inference_server.log" source ../common/util.sh # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="onnx savedmodel plan libtorch"} +BACKENDS=${BACKENDS:="onnx plan libtorch"} rm -f $SERVER_LOG $CLIENT_LOG diff --git a/qa/L0_batcher/batcher_test.py b/qa/L0_batcher/batcher_test.py index 38e208c21e..780eb1e68f 100755 --- a/qa/L0_batcher/batcher_test.py +++ b/qa/L0_batcher/batcher_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -61,7 +61,7 @@ USE_GRPC = False assert USE_GRPC or USE_HTTP, "USE_GRPC or USE_HTTP must be non-zero" -BACKENDS = os.environ.get("BACKENDS", "graphdef savedmodel onnx libtorch plan python") +BACKENDS = os.environ.get("BACKENDS", "onnx libtorch plan python") _trials = BACKENDS.split(" ") @@ -153,9 +153,7 @@ def check_response( start_ms = int(round(time.time() * 1000)) if ( - trial == "savedmodel" - or trial == "graphdef" - or trial == "libtorch" + trial == "libtorch" or trial == "onnx" or trial == "plan" or trial == "python" diff --git a/qa/L0_batcher/test.sh b/qa/L0_batcher/test.sh index 2eed5e3f13..136fbe586d 100755 --- a/qa/L0_batcher/test.sh +++ b/qa/L0_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -74,8 +74,6 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then test_multi_batch_different_shape_allow_ragged" fi -TF_VERSION=${TF_VERSION:=2} - # On windows the paths invoked by the script (running in WSL) must use # /mnt/c when needed but the paths on the tritonserver command-line # must be C:/ style. @@ -105,13 +103,13 @@ else fi fi -SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR} --backend-config=tensorflow,version=${TF_VERSION}" +SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR}" source ../common/util.sh RET=0 # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan python"} +BACKENDS=${BACKENDS:="onnx libtorch plan python"} export BACKENDS # Basic batcher tests diff --git a/qa/L0_client_nobatch/client_test.py b/qa/L0_client_nobatch/client_test.py index 3288fc2ebf..f2e0fc398b 100755 --- a/qa/L0_client_nobatch/client_test.py +++ b/qa/L0_client_nobatch/client_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -43,12 +43,12 @@ class ClientNoBatchTest(tu.TestResultCollector): def test_nobatch_request_for_batching_model(self): input_size = 16 - # graphdef_int32_int8_int8 has a batching version with max batch size of 8. + # onnx_int32_int8_int8 has a batching version with max batch size of 8. # The server should return an error if the batch size is not included in the # input shapes. tensor_shape = (input_size,) for protocol in ["http", "grpc"]: - model_name = tu.get_model_name("graphdef", np.int32, np.int8, np.int8) + model_name = tu.get_model_name("onnx", np.int32, np.int8, np.int8) in0 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) in1 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) @@ -94,14 +94,12 @@ def test_nobatch_request_for_batching_model(self): def test_batch_request_for_nobatching_model(self): input_size = 16 - # graphdef_nobatch_int32_int8_int8 is non batching version. + # onnx_nobatch_int32_int8_int8 is non batching version. # The server should return an error if the batch size dimension # is included in the shape tensor_shape = (1, input_size) for protocol in ["http", "grpc"]: - model_name = tu.get_model_name( - "graphdef_nobatch", np.int32, np.int8, np.int8 - ) + model_name = tu.get_model_name("onnx_nobatch", np.int32, np.int8, np.int8) in0 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) in1 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) @@ -148,14 +146,12 @@ def test_batch_request_for_nobatching_model(self): def test_nobatch_request_for_nonbatching_model(self): input_size = 16 - # graphdef_int32_int8_int8 has a batching version with max batch size of 8. + # onnx_int32_int8_int8 has a batching version with max batch size of 8. # The server should return an error if the batch size is not included in the # input shapes. tensor_shape = (input_size,) for protocol in ["http", "grpc"]: - model_name = tu.get_model_name( - "graphdef_nobatch", np.int32, np.int8, np.int8 - ) + model_name = tu.get_model_name("onnx_nobatch", np.int32, np.int8, np.int8) in0 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) in1 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) @@ -195,12 +191,12 @@ def test_nobatch_request_for_nonbatching_model(self): def test_batch_request_for_batching_model(self): input_size = 16 - # graphdef_nobatch_int32_int8_int8 is non batching version. + # onnx_nobatch_int32_int8_int8 is non batching version. # The server should return an error if the batch size dimension # is included in the shape tensor_shape = (1, input_size) for protocol in ["http", "grpc"]: - model_name = tu.get_model_name("graphdef", np.int32, np.int8, np.int8) + model_name = tu.get_model_name("onnx", np.int32, np.int8, np.int8) in0 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) in1 = np.random.randint(low=0, high=100, size=tensor_shape, dtype=np.int32) diff --git a/qa/L0_client_nobatch/test.sh b/qa/L0_client_nobatch/test.sh index e768f385e5..cb1a05d660 100755 --- a/qa/L0_client_nobatch/test.sh +++ b/qa/L0_client_nobatch/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,7 @@ EXPECTED_NUM_TESTS="4" DATADIR=/data/inferenceserver/${REPO_VERSION} MODELDIR="${PWD}/qa_model_repository" -rm -rf ${MODELDIR} && cp -r "${DATADIR}/qa_model_repository" ${MODELDIR} +rm -rf ${MODELDIR} && cp -r "${DATADIR}/qa_model_repository/onnx_*" ${MODELDIR} # Note there is a coupling in ./client_test.py SERVER=/opt/tritonserver/bin/tritonserver SERVER_ARGS="--model-repository=${MODELDIR}" SERVER_LOG="./inference_server.log" diff --git a/qa/L0_compute_capability/test.sh b/qa/L0_compute_capability/test.sh index d85acb1b6e..066b8fbf23 100755 --- a/qa/L0_compute_capability/test.sh +++ b/qa/L0_compute_capability/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,7 +50,7 @@ rm -f *.log RET=0 # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan"} +BACKENDS=${BACKENDS:="onnx libtorch plan"} for BACKEND in $BACKENDS; do # Need just one model for the backend... @@ -67,8 +67,8 @@ for BACKEND in $BACKENDS; do # Run with a high minimum capability so that no GPUs are # recognized. This should cause the server to fail to start since # we explicitly asked for a GPU in the instance_group. - SERVER_ARGS="--min-supported-compute-capability=100.0 --model-repository=`pwd`/models" - SERVER_LOG="./inference_server_${BACKEND}_cc100.log" + SERVER_ARGS="--min-supported-compute-capability=900.0 --model-repository=`pwd`/models" + SERVER_LOG="./inference_server_${BACKEND}_cc900.log" run_server if [ "$SERVER_PID" != "0" ]; then echo -e "\n***\n*** Unexpected success with min compute 100.0 for ${BACKEND}\n***" diff --git a/qa/L0_config_json/ensemble_config.pbtxt b/qa/L0_config_json/ensemble_config.pbtxt index 29de01a3aa..b1fd8972b5 100644 --- a/qa/L0_config_json/ensemble_config.pbtxt +++ b/qa/L0_config_json/ensemble_config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ output [ ensemble_scheduling { step [ { - model_name: "savedmodel_nobatch_float32_float32_float32" + model_name: "onnx_nobatch_float32_float32_float32" model_version: 1 input_map [ { @@ -78,7 +78,7 @@ ensemble_scheduling { ] }, { - model_name: "savedmodel_nobatch_float32_float32_float32" + model_name: "onnx_nobatch_float32_float32_float32" model_version: -1 input_map [ { diff --git a/qa/L0_config_json/test.sh b/qa/L0_config_json/test.sh index b1016b806b..7d073d7bb2 100755 --- a/qa/L0_config_json/test.sh +++ b/qa/L0_config_json/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -52,7 +52,7 @@ RET=0 rm -fr *.log rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. # Test input and output dims are shown as numbers TRIAL=ios @@ -65,7 +65,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out @@ -87,8 +87,8 @@ wait $SERVER_PID TRIAL=reshape rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +(cd models/onnx_nobatch_float32_float32_float32 && \ sed -i "s/data_type:.*TYPE_FP32/data_type: TYPE_FP32\nreshape: { shape: [ 16 ]}/g" config.pbtxt) run_server @@ -99,7 +99,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out @@ -121,8 +121,8 @@ wait $SERVER_PID TRIAL=specific rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +(cd models/onnx_nobatch_float32_float32_float32 && \ sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [1] }}/" config.pbtxt) run_server @@ -133,7 +133,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out @@ -157,8 +157,8 @@ wait $SERVER_PID TRIAL=dbatch rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +(cd models/onnx_nobatch_float32_float32_float32 && \ echo "dynamic_batching: { max_queue_delay_microseconds: 42 \ default_queue_policy: { default_timeout_microseconds: 123 } \ priority_queue_policy: { key: 1 value: { default_timeout_microseconds: 123 }} \ @@ -172,7 +172,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out @@ -209,8 +209,8 @@ wait $SERVER_PID TRIAL=sbatch rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +(cd models/onnx_nobatch_float32_float32_float32 && \ echo "sequence_batching: { max_sequence_idle_microseconds: 42 \ oldest: { max_queue_delay_microseconds: 987 }}" >> config.pbtxt) @@ -222,7 +222,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out @@ -265,7 +265,7 @@ wait $SERVER_PID TRIAL=ensemble rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. mkdir -p models/simple_ensemble/1 && cp ensemble_config.pbtxt models/simple_ensemble/config.pbtxt run_server @@ -307,8 +307,8 @@ rm -fr models/simple_ensemble TRIAL=warmup rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +(cd models/onnx_nobatch_float32_float32_float32 && \ echo "model_warmup [{" >> config.pbtxt && \ echo " name : \"warmup 1\"" >> config.pbtxt && \ echo " batch_size: 1" >> config.pbtxt && \ @@ -355,7 +355,7 @@ if [ "$SERVER_PID" == "0" ]; then fi set +e -code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` +code=`curl -s -w %{http_code} -o ./$TRIAL.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $TRIAL.out diff --git a/qa/L0_custom_model_config/test.sh b/qa/L0_custom_model_config/test.sh index d839cacbd5..99e9ed8b10 100755 --- a/qa/L0_custom_model_config/test.sh +++ b/qa/L0_custom_model_config/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -51,8 +51,8 @@ RET=0 rm -fr *.log rm -fr models && mkdir models -cp -r $DATADIR/qa_model_repository/savedmodel_nobatch_float32_float32_float32 models/. -mkdir models/savedmodel_nobatch_float32_float32_float32/configs +cp -r $DATADIR/qa_model_repository/onnx_nobatch_float32_float32_float32 models/. +mkdir models/onnx_nobatch_float32_float32_float32/configs test_custom_config() { @@ -66,7 +66,7 @@ test_custom_config() fi set +e - code=`curl -s -w %{http_code} -o ./curl.out localhost:8000/v2/models/savedmodel_nobatch_float32_float32_float32/config` + code=`curl -s -w %{http_code} -o ./curl.out localhost:8000/v2/models/onnx_nobatch_float32_float32_float32/config` set -e if [ "$code" != "200" ]; then cat $out.out @@ -92,15 +92,15 @@ VERSION_V100="2" VERSION_CUSTOM="3" # Distinguish configs with different model versions -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +(cd models/onnx_nobatch_float32_float32_float32 && \ sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_DEFAULT] }}/" config.pbtxt) -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +(cd models/onnx_nobatch_float32_float32_float32 && \ cp config.pbtxt configs/h100.pbtxt && \ sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_H100] }}/" configs/h100.pbtxt) -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +(cd models/onnx_nobatch_float32_float32_float32 && \ cp config.pbtxt configs/v100.pbtxt && \ sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_V100] }}/" configs/v100.pbtxt) -(cd models/savedmodel_nobatch_float32_float32_float32 && \ +(cd models/onnx_nobatch_float32_float32_float32 && \ cp config.pbtxt configs/config.pbtxt && \ sed -i "s/^version_policy:.*/version_policy: { specific: { versions: [$VERSION_CUSTOM] }}/" configs/config.pbtxt) diff --git a/qa/L0_custom_ops/cuda_op_test.py b/qa/L0_custom_ops/cuda_op_test.py deleted file mode 100755 index bc610b3f0b..0000000000 --- a/qa/L0_custom_ops/cuda_op_test.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse -import sys -from builtins import range - -import numpy as np -import tritonclient.grpc as grpcclient -import tritonclient.http as httpclient -from tritonclient.utils import np_to_triton_dtype - -FLAGS = None - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-v", - "--verbose", - action="store_true", - required=False, - default=False, - help="Enable verbose output", - ) - parser.add_argument( - "-u", - "--url", - type=str, - required=False, - default="localhost:8000", - help="Inference server URL. Default is localhost:8000.", - ) - parser.add_argument( - "-i", - "--protocol", - type=str, - required=False, - default="http", - help='Protocol ("http"/"grpc") used to ' - + 'communicate with inference service. Default is "http".', - ) - parser.add_argument("-m", "--model", type=str, required=True, help="Name of model.") - - FLAGS = parser.parse_args() - if (FLAGS.protocol != "http") and (FLAGS.protocol != "grpc"): - print( - 'unexpected protocol "{}", expects "http" or "grpc"'.format(FLAGS.protocol) - ) - exit(1) - - client_util = httpclient if FLAGS.protocol == "http" else grpcclient - - # Run the cudaop model, which depends on a custom operation that - # uses CUDA. The custom operator adds one to each input - model_name = FLAGS.model - elements = 8 - - # Create the inference context for the model. - client = client_util.InferenceServerClient(FLAGS.url, verbose=FLAGS.verbose) - - # Create the data for one input tensor. - input_data = np.arange(start=42, stop=42 + elements, dtype=np.int32) - - inputs = [ - client_util.InferInput( - "in", input_data.shape, np_to_triton_dtype(input_data.dtype) - ) - ] - inputs[0].set_data_from_numpy(input_data) - - results = client.infer(model_name, inputs) - output_data = results.as_numpy("out") - if output_data is None: - print("error: expected 'out'") - sys.exit(1) - - for i in range(elements): - print( - str(i) + ": input " + str(input_data[i]) + ", output " + str(output_data[i]) - ) - if output_data[i] != (input_data[i] + 1): - print("error: incorrect value") - sys.exit(1) diff --git a/qa/L0_custom_ops/test.sh b/qa/L0_custom_ops/test.sh index f3b792e59a..60cb5d7a82 100755 --- a/qa/L0_custom_ops/test.sh +++ b/qa/L0_custom_ops/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,8 +41,6 @@ fi export CUDA_VISIBLE_DEVICES=0 CLIENT_LOG="./client.log" -ZERO_OUT_TEST=zero_out_test.py -CUDA_OP_TEST=cuda_op_test.py MOD_OP_TEST=mod_op_test.py VISION_OP_TEST=vision_op_test.py ONNX_OP_TEST=onnx_op_test.py @@ -55,113 +53,6 @@ rm -f $SERVER_LOG $CLIENT_LOG RET=0 -# Must explicitly set LD_LIBRARY_PATH so that the custom operations -# can find libtensorflow_framework.so. -LD_LIBRARY_PATH=/opt/tritonserver/backends/tensorflow:$LD_LIBRARY_PATH - -# Tensorflow -## Load operations via LD_PRELOAD -SERVER_ARGS="--model-repository=/data/inferenceserver/${REPO_VERSION}/qa_custom_ops/tf_custom_ops" -SERVER_LD_PRELOAD="/data/inferenceserver/${REPO_VERSION}/qa_custom_ops/tf_custom_ops/libzeroout.so:/data/inferenceserver/${REPO_VERSION}/qa_custom_ops/tf_custom_ops/libcudaop.so:/data/inferenceserver/${REPO_VERSION}/qa_custom_ops/tf_custom_ops/libbusyop.so" - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e - -python $ZERO_OUT_TEST -v -m graphdef_zeroout >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $ZERO_OUT_TEST -v -m savedmodel_zeroout >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $CUDA_OP_TEST -v -m graphdef_cudaop >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $CUDA_OP_TEST -v -m savedmodel_cudaop >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -set -e - -kill $SERVER_PID -wait $SERVER_PID - -## Load operations via model config -SERVER_ARGS="--model-repository=tf_custom_ops" -SERVER_LD_PRELOAD="" - -rm -rf tf_custom_ops && \ - mkdir -p tf_custom_ops && \ - cp -r /data/inferenceserver/${REPO_VERSION}/qa_custom_ops/tf_custom_ops . - -for MODEL_TYPE in savedmodel graphdef; do - echo "model_operations { op_library_filename: \"tf_custom_ops/libbusyop.so\" }" >> tf_custom_ops/${MODEL_TYPE}_busyop/config.pbtxt - echo "model_operations { op_library_filename: \"tf_custom_ops/libcudaop.so\" }" >> tf_custom_ops/${MODEL_TYPE}_cudaop/config.pbtxt - echo "model_operations { op_library_filename: \"tf_custom_ops/libzeroout.so\" }" >> tf_custom_ops/${MODEL_TYPE}_zeroout/config.pbtxt -done - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e - -python $ZERO_OUT_TEST -v -m graphdef_zeroout >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $ZERO_OUT_TEST -v -m savedmodel_zeroout >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $CUDA_OP_TEST -v -m graphdef_cudaop >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -python $CUDA_OP_TEST -v -m savedmodel_cudaop >>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -fi - -set -e - -kill $SERVER_PID -wait $SERVER_PID - # Must set LD_LIBRARY_PATH just for the server launch so that the # custom operations can find libtorch.so and other pytorch dependencies. LD_LIBRARY_PATH=/opt/tritonserver/backends/pytorch:$LD_LIBRARY_PATH diff --git a/qa/L0_custom_ops/zero_out_test.py b/qa/L0_custom_ops/zero_out_test.py deleted file mode 100755 index 86fdcb8a30..0000000000 --- a/qa/L0_custom_ops/zero_out_test.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import argparse -import sys -from builtins import range - -import numpy as np -import tritonclient.grpc as grpcclient -import tritonclient.http as httpclient -from tritonclient.utils import np_to_triton_dtype - -FLAGS = None - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-v", - "--verbose", - action="store_true", - required=False, - default=False, - help="Enable verbose output", - ) - parser.add_argument( - "-u", - "--url", - type=str, - required=False, - default="localhost:8000", - help="Inference server URL. Default is localhost:8000.", - ) - parser.add_argument( - "-i", - "--protocol", - type=str, - required=False, - default="http", - help='Protocol ("http"/"grpc") used to ' - + 'communicate with inference service. Default is "http".', - ) - parser.add_argument("-m", "--model", type=str, required=True, help="Name of model.") - - FLAGS = parser.parse_args() - if (FLAGS.protocol != "http") and (FLAGS.protocol != "grpc"): - print( - 'unexpected protocol "{}", expects "http" or "grpc"'.format(FLAGS.protocol) - ) - exit(1) - - client_util = httpclient if FLAGS.protocol == "http" else grpcclient - - # Run the zero-out model, which depends on a custom operation - model_name = FLAGS.model - elements = 8 - - # Create the inference context for the model. - client = client_util.InferenceServerClient(FLAGS.url, verbose=FLAGS.verbose) - - # Create the data for one input tensor. - input_data = np.arange(start=42, stop=42 + elements, dtype=np.int32) - - inputs = [ - client_util.InferInput( - "to_zero", input_data.shape, np_to_triton_dtype(input_data.dtype) - ) - ] - inputs[0].set_data_from_numpy(input_data) - results = client.infer(model_name, inputs) - - # We expect 1 result with all inputs except first to be zeroed. - output_data = results.as_numpy("zeroed") - if output_data is None: - print("error: expected 'zeroed'") - sys.exit(1) - - for i in range(elements): - print( - str(i) + ": input " + str(input_data[i]) + ", output " + str(output_data[i]) - ) - if (i == 0) and (input_data[i] != output_data[i]): - print("error: incorrect value") - sys.exit(1) - if (i != 0) and (output_data[i] != 0): - print("error: expected 0") - sys.exit(1) diff --git a/qa/L0_dlpack_multi_gpu/test.sh b/qa/L0_dlpack_multi_gpu/test.sh index a90169780c..d85a089431 100755 --- a/qa/L0_dlpack_multi_gpu/test.sh +++ b/qa/L0_dlpack_multi_gpu/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,7 +42,6 @@ source ../common/util.sh # Uninstall the non CUDA version of PyTorch pip3 uninstall -y torch pip3 install torch==2.3.1+cu118 -f https://download.pytorch.org/whl/torch_stable.html -pip3 install tensorflow # Install CuPy for testing non_blocking compute streams pip3 install cupy-cuda12x diff --git a/qa/L0_dyna_sequence_batcher/dyna_sequence_batcher_test.py b/qa/L0_dyna_sequence_batcher/dyna_sequence_batcher_test.py index f2c709469b..47a0e2da32 100755 --- a/qa/L0_dyna_sequence_batcher/dyna_sequence_batcher_test.py +++ b/qa/L0_dyna_sequence_batcher/dyna_sequence_batcher_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -44,9 +44,7 @@ _test_cuda_shared_memory = bool(int(os.environ.get("TEST_CUDA_SHARED_MEMORY", 0))) NO_BATCHING = int(os.environ.get("NO_BATCHING", 0)) == 1 -BACKENDS = os.environ.get( - "BACKENDS", "graphdef savedmodel libtorch onnx plan custom custom_string" -) +BACKENDS = os.environ.get("BACKENDS", "libtorch onnx plan custom custom_string") IMPLICIT_STATE = int(os.environ["IMPLICIT_STATE"]) == 1 _trials = BACKENDS.split(" ") @@ -74,7 +72,6 @@ def get_expected_result(self, expected_result, corrid, value, trial, flag_str=No # information. if ( (("nobatch" not in trial) and ("custom" not in trial)) - or ("graphdef" in trial) or ("plan" in trial) or ("onnx" in trial) or ("libtorch" in trial) diff --git a/qa/L0_dyna_sequence_batcher/test.sh b/qa/L0_dyna_sequence_batcher/test.sh index d545886293..afa2bb35f9 100755 --- a/qa/L0_dyna_sequence_batcher/test.sh +++ b/qa/L0_dyna_sequence_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -53,7 +53,7 @@ IMPLICIT_STATE=${IMPLICIT_STATE:="0"} export IMPLICIT_STATE # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel libtorch onnx plan custom custom_string"} +BACKENDS=${BACKENDS:="libtorch onnx plan custom custom_string"} export BACKENDS MODEL_REPOSITORY='' diff --git a/qa/L0_grpc/test.sh b/qa/L0_grpc/test.sh index 93d22e75be..28d9236d0a 100755 --- a/qa/L0_grpc/test.sh +++ b/qa/L0_grpc/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -149,7 +149,7 @@ cp -r ${MODELDIR}/simple_dyna_sequence ${MODELDIR}/simple_string_dyna_sequence sed -i "s/simple_dyna_sequence/simple_string_dyna_sequence/g" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt sed -i "s/^platform: .*/backend: \"dyna_sequence\"/g" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt sed -i "/CONTROL_SEQUENCE_CORRID/{n;s/data_type:.*/data_type: TYPE_STRING/}" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt -rm -f ${MODELDIR}/simple_string_dyna_sequence/1/model.graphdef +rm -f ${MODELDIR}/simple_string_dyna_sequence/1/model.onnx cp ../custom_models/custom_dyna_sequence_int32/1/libtriton_dyna_sequence.so ${MODELDIR}/simple_string_dyna_sequence/1/ rm -f *.log @@ -205,19 +205,19 @@ for i in \ EXTRA_ARGS="-i grpc -u localhost:8001" fi if [[ ($SUFFIX == "image_client") || ($SUFFIX == "grpc_image_client") ]]; then - python $i -m inception_graphdef -s INCEPTION -a -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.async.${SUFFIX} RET=1 fi - python $i -m inception_graphdef -s INCEPTION -a --streaming -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.streaming.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a --streaming -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.streaming.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.streaming.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.streaming.${SUFFIX} RET=1 fi - python $i -m inception_graphdef -s INCEPTION -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -c 1 -b 1 $EXTRA_ARGS $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.${SUFFIX} @@ -270,19 +270,19 @@ for i in \ BASE=$(basename -- $i) SUFFIX="${BASE%.*}" if [[ $SUFFIX == "image_client" ]]; then - $i -m inception_graphdef -s INCEPTION -a -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.async.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} RET=1 fi - $i -m inception_graphdef -s INCEPTION -a --streaming -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.streaming.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -a --streaming -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.streaming.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.streaming.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} RET=1 fi - $i -m inception_graphdef -s INCEPTION -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.${SUFFIX}" 2>&1 + $i -m inception_onnx -s INCEPTION -c 1 -b 1 -i grpc -u localhost:8001 $IMAGE >> "${CLIENT_LOG}.c++.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.c++.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.c++.${SUFFIX} diff --git a/qa/L0_http/test.sh b/qa/L0_http/test.sh index c90c2e5010..c36024e007 100755 --- a/qa/L0_http/test.sh +++ b/qa/L0_http/test.sh @@ -119,7 +119,7 @@ cp -r ${MODELDIR}/simple_dyna_sequence ${MODELDIR}/simple_string_dyna_sequence sed -i "s/simple_dyna_sequence/simple_string_dyna_sequence/g" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt sed -i "s/^platform: .*/backend: \"dyna_sequence\"/g" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt sed -i "/CONTROL_SEQUENCE_CORRID/{n;s/data_type:.*/data_type: TYPE_STRING/}" ${MODELDIR}/simple_string_dyna_sequence/config.pbtxt -rm -f ${MODELDIR}/simple_string_dyna_sequence/1/model.graphdef +rm -f ${MODELDIR}/simple_string_dyna_sequence/1/model.onnx cp ../custom_models/custom_dyna_sequence_int32/1/libtriton_dyna_sequence.so ${MODELDIR}/simple_string_dyna_sequence/1/ rm -f *.log @@ -163,13 +163,13 @@ for i in \ BASE=$(basename -- $i) SUFFIX="${BASE%.*}" if [ $SUFFIX == "image_client" ]; then - python $i -m inception_graphdef -s INCEPTION -a -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -a -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.async.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.async.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.async.${SUFFIX} RET=1 fi - python $i -m inception_graphdef -s INCEPTION -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 + python $i -m inception_onnx -s INCEPTION -c 1 -b 1 $IMAGE >> "${CLIENT_LOG}.${SUFFIX}" 2>&1 if [ `grep -c VULTURE ${CLIENT_LOG}.${SUFFIX}` != "1" ]; then echo -e "\n***\n*** Failed. Expected 1 VULTURE results\n***" cat $CLIENT_LOG.${SUFFIX} diff --git a/qa/L0_http_fuzz/test.sh b/qa/L0_http_fuzz/test.sh index 30cf02bc2a..1abe08b487 100755 --- a/qa/L0_http_fuzz/test.sh +++ b/qa/L0_http_fuzz/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -46,7 +46,7 @@ rm -f *.log *.db EXPECTED_NUM_TESTS="1" mkdir -p models -cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/savedmodel_zero_1_object models/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/onnx_zero_1_object models/ FUZZTEST=fuzztest.py FUZZ_LOG=`pwd`/fuzz.log diff --git a/qa/L0_infer/infer_test.py b/qa/L0_infer/infer_test.py index 63433f01c9..8f16a7fe10 100755 --- a/qa/L0_infer/infer_test.py +++ b/qa/L0_infer/infer_test.py @@ -48,7 +48,7 @@ assert USE_GRPC or USE_HTTP, "USE_GRPC or USE_HTTP must be non-zero" BACKENDS = os.environ.get( - "BACKENDS", "graphdef savedmodel onnx libtorch plan python python_dlpack openvino" + "BACKENDS", "onnx libtorch plan python python_dlpack openvino" ) ENSEMBLES = bool(int(os.environ.get("ENSEMBLES", 1))) NOBATCH = bool(int(os.environ.get("NOBATCH", 1))) @@ -163,7 +163,7 @@ def _infer_exact_helper( ): ensemble_prefix.append(prefix) - if tu.validate_for_tf_model( + if tu.validate_for_onnx_model( input_dtype, output0_dtype, output1_dtype, @@ -172,7 +172,7 @@ def _infer_exact_helper( (input_size,), ): for prefix in ensemble_prefix: - for pf in ["graphdef", "savedmodel"]: + for pf in ["onnx"]: if pf in BACKENDS: _infer_exact_helper( self, @@ -670,9 +670,9 @@ def test_raw_version_latest_1(self): input_size = 16 tensor_shape = (1, input_size) - # There are 3 versions of graphdef_int8_int8_int8 but + # There are 3 versions of onnx_int8_int8_int8 but # only version 3 should be available - for platform in ("graphdef", "savedmodel"): + for platform in ["onnx"]: if platform not in BACKENDS: continue try: @@ -733,9 +733,9 @@ def test_raw_version_latest_2(self): input_size = 16 tensor_shape = (1, input_size) - # There are 3 versions of graphdef_int16_int16_int16 but only + # There are 3 versions of onnx_int16_int16_int16 but only # versions 2 and 3 should be available - for platform in ("graphdef", "savedmodel"): + for platform in ["onnx"]: if platform not in BACKENDS: continue try: @@ -794,7 +794,7 @@ def test_raw_version_all(self): # There are 3 versions of *_int32_int32_int32 and all should # be available. - for platform in ("graphdef", "savedmodel"): + for platform in ["onnx"]: if platform not in BACKENDS: continue iu.infer_exact( @@ -849,7 +849,7 @@ def test_raw_version_specific_1(self): # There are 3 versions of *_float16_float16_float16 but only # version 1 should be available. - for platform in ("graphdef", "savedmodel"): + for platform in ["onnx"]: if platform not in BACKENDS: continue iu.infer_exact( @@ -911,7 +911,7 @@ def test_raw_version_specific_1_3(self): # There are 3 versions of *_float32_float32_float32 but only # versions 1 and 3 should be available. - for platform in ("graphdef", "savedmodel", "plan"): + for platform in ("onnx", "plan"): if platform == "plan" and CPU_ONLY: continue if platform not in BACKENDS: @@ -969,7 +969,7 @@ def test_raw_version_specific_1_3(self): ) if ENSEMBLES: - if all(x in BACKENDS for x in ["graphdef", "savedmodel"]): + if all(x in BACKENDS for x in ["onnx", "plan"]): def test_ensemble_mix_platform(self): # Skip on CPU only machine as TensorRT model is used in this ensemble @@ -990,7 +990,7 @@ def test_ensemble_mix_platform(self): use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if "graphdef" in BACKENDS: + if "onnx" in BACKENDS: def test_ensemble_mix_type(self): for bs in (1, 8): @@ -1008,7 +1008,7 @@ def test_ensemble_mix_type(self): use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if all(x in BACKENDS for x in ["graphdef", "savedmodel"]): + if all(x in BACKENDS for x in ["onnx", "plan"]): def test_ensemble_mix_ensemble(self): for bs in (1, 8): @@ -1029,7 +1029,7 @@ def test_ensemble_mix_ensemble(self): if all( x in BACKENDS for x in [ - "graphdef", + "onnx", ] ): @@ -1083,7 +1083,7 @@ def test_ensemble_mix_batch_nobatch(self): if not (TEST_SYSTEM_SHARED_MEMORY or TEST_CUDA_SHARED_MEMORY): def test_ensemble_label_lookup(self): - if all(x in BACKENDS for x in ["graphdef", "savedmodel"]): + if all(x in BACKENDS for x in ["onnx", "plan"]): # Ensemble needs to look up label from the actual model for bs in (1, 8): iu.infer_exact( @@ -1102,7 +1102,7 @@ def test_ensemble_label_lookup(self): use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if all(x in BACKENDS for x in ["graphdef", "savedmodel"]): + if all(x in BACKENDS for x in ["onnx", "plan"]): # Label from the actual model will be passed along the nested ensemble for bs in (1, 8): iu.infer_exact( @@ -1121,7 +1121,7 @@ def test_ensemble_label_lookup(self): use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, ) - if "graphdef" in BACKENDS: + if "onnx" in BACKENDS: # If label file is provided, it will use the provided label file directly try: iu.infer_exact( @@ -1144,7 +1144,7 @@ def test_ensemble_label_lookup(self): # with unexpected labels pass - if "graphdef" in BACKENDS: + if "onnx" in BACKENDS: for bs in (1, 8): iu.infer_exact( self, diff --git a/qa/L0_infer/test.sh b/qa/L0_infer/test.sh index 2d54fd8965..79ba093ffd 100755 --- a/qa/L0_infer/test.sh +++ b/qa/L0_infer/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -67,16 +67,15 @@ if [ "$TEST_VALGRIND" -eq 1 ]; then rm -f $LEAKCHECK_LOG_BASE* # Remove 'python', 'python_dlpack' and 'onnx' from BACKENDS and test them # separately below. - BACKENDS="graphdef savedmodel libtorch plan openvino" + BACKENDS="libtorch plan openvino" fi if [ "$TEST_SYSTEM_SHARED_MEMORY" -eq 1 ] || [ "$TEST_CUDA_SHARED_MEMORY" -eq 1 ]; then EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="33"} else - EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="46"} + EXPECTED_NUM_TESTS=${EXPECTED_NUM_TESTS:="44"} fi -TF_VERSION=${TF_VERSION:=2} TEST_JETSON=${TEST_JETSON:=0} # Default size (in MB) of shared memory to be used by each python model @@ -109,7 +108,7 @@ else fi # Allow more time to exit. Ensemble brings in too many models -SERVER_ARGS_EXTRA="--exit-timeout-secs=${SERVER_TIMEOUT} --backend-directory=${BACKEND_DIR} --backend-config=tensorflow,version=${TF_VERSION} --backend-config=python,stub-timeout-seconds=120 --backend-config=python,shm-default-byte-size=${DEFAULT_SHM_SIZE_BYTES}" +SERVER_ARGS_EXTRA="--exit-timeout-secs=${SERVER_TIMEOUT} --backend-directory=${BACKEND_DIR} --backend-config=python,stub-timeout-seconds=120 --backend-config=python,shm-default-byte-size=${DEFAULT_SHM_SIZE_BYTES}" SERVER_ARGS="--model-repository=${MODELDIR} ${SERVER_ARGS_EXTRA}" SERVER_LOG_BASE="./inference_server" source ../common/util.sh @@ -129,7 +128,7 @@ if [ "$TRITON_SERVER_CPU_ONLY" == "1" ]; then fi # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan python python_dlpack openvino"} +BACKENDS=${BACKENDS:="onnx libtorch plan python python_dlpack openvino"} export BACKENDS # If ENSEMBLES not specified, set to 1 @@ -234,12 +233,12 @@ function generate_model_repository() { create_nop_version_dir `pwd`/models - if [[ $BACKENDS == *"graphdef"* ]]; then + if [[ $BACKENDS == *"onnx"* ]]; then ENSEMBLE_MODELS="wrong_label_int32_float32_float32 label_override_int32_float32_float32 mix_type_int32_float32_float32" ENSEMBLE_MODELS="${ENSEMBLE_MODELS} batch_to_nobatch_float32_float32_float32 batch_to_nobatch_nobatch_float32_float32_float32 nobatch_to_batch_float32_float32_float32 nobatch_to_batch_nobatch_float32_float32_float32 mix_nobatch_batch_float32_float32_float32" - if [[ $BACKENDS == *"savedmodel"* ]] ; then + if [[ $BACKENDS == *"libtorch"* ]] ; then ENSEMBLE_MODELS="${ENSEMBLE_MODELS} mix_platform_float32_float32_float32 mix_ensemble_int32_float32_float32" fi diff --git a/qa/L0_infer_reshape/infer_reshape_test.py b/qa/L0_infer_reshape/infer_reshape_test.py index e77dcbecaf..5445f5d6c9 100755 --- a/qa/L0_infer_reshape/infer_reshape_test.py +++ b/qa/L0_infer_reshape/infer_reshape_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -50,68 +50,6 @@ def _full_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True): output_shapes = input_shapes # For validation assume any shape can be used... - if tu.validate_for_tf_model( - dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] - ): - # model that supports batching - for bs in (1, 8): - full_shapes = [ - [ - bs, - ] - + input_shape - for input_shape in input_shapes - ] - full_output_shapes = [ - [ - bs, - ] - + output_shape - for output_shape in output_shapes - ] - iu.infer_zero( - self, - "graphdef", - bs, - dtype, - full_shapes, - full_output_shapes, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - iu.infer_zero( - self, - "savedmodel", - bs, - dtype, - full_shapes, - full_output_shapes, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - # model that does not support batching - if no_batch: - iu.infer_zero( - self, - "graphdef_nobatch", - 1, - dtype, - input_shapes, - output_shapes, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - iu.infer_zero( - self, - "savedmodel_nobatch", - 1, - dtype, - input_shapes, - output_shapes, - use_system_shared_memory=TEST_SYSTEM_SHARED_MEMORY, - use_cuda_shared_memory=TEST_CUDA_SHARED_MEMORY, - ) - if tu.validate_for_onnx_model( dtype, dtype, dtype, input_shapes[0], input_shapes[0], input_shapes[0] ): diff --git a/qa/L0_infer_reshape/test.sh b/qa/L0_infer_reshape/test.sh index 218be954d9..249a1c0ca4 100755 --- a/qa/L0_infer_reshape/test.sh +++ b/qa/L0_infer_reshape/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -67,7 +67,7 @@ for i in \ zero_1_int32 \ zero_2_int32 \ zero_3_int32 ; do - cp -r models/graphdef_${i} models/custom_${i} + cp -r models/onnx_${i} models/custom_${i} rm -fr models/custom_${i}/1/* (cd models/custom_${i} && \ sed -i "s/^platform:.*/backend: \"identity\"/" config.pbtxt && \ diff --git a/qa/L0_infer_variable/infer_variable_test.py b/qa/L0_infer_variable/infer_variable_test.py index e5e6470a3c..55a2c7a084 100755 --- a/qa/L0_infer_variable/infer_variable_test.py +++ b/qa/L0_infer_variable/infer_variable_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -146,29 +146,6 @@ def _infer_exact_helper( ): ensemble_prefix.append(prefix) - if tu.validate_for_tf_model( - input_dtype, - output0_dtype, - output1_dtype, - input_shape, - output0_shape, - output1_shape, - ): - for prefix in ensemble_prefix: - for pf in ["graphdef", "savedmodel"]: - _infer_exact_helper( - self, - prefix + pf, - input_shape, - 8, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - swap=swap, - ) - if tu.validate_for_trt_model( input_dtype, output0_dtype, diff --git a/qa/L0_infer_variable/test.sh b/qa/L0_infer_variable/test.sh index 9760583b94..36d86716ad 100755 --- a/qa/L0_infer_variable/test.sh +++ b/qa/L0_infer_variable/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -68,7 +68,7 @@ for TARGET in cpu gpu; do KIND="KIND_GPU" && [[ "$TARGET" == "cpu" ]] && KIND="KIND_CPU" # Onnx models are handled separately, see below - for FW in graphdef savedmodel onnx libtorch; do + for FW in onnx libtorch; do for MC in `ls models/${FW}*/config.pbtxt`; do echo "instance_group [ { kind: ${KIND} }]" >> $MC done diff --git a/qa/L0_infer_zero/infer_zero_test.py b/qa/L0_infer_zero/infer_zero_test.py index 3786c5b4a1..d05b383cff 100755 --- a/qa/L0_infer_zero/infer_zero_test.py +++ b/qa/L0_infer_zero/infer_zero_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -41,11 +41,9 @@ TEST_SYSTEM_SHARED_MEMORY = bool(int(os.environ.get("TEST_SYSTEM_SHARED_MEMORY", 0))) TEST_CUDA_SHARED_MEMORY = bool(int(os.environ.get("TEST_CUDA_SHARED_MEMORY", 0))) -BACKENDS = os.environ.get("BACKENDS", "graphdef savedmodel onnx libtorch") +BACKENDS = os.environ.get("BACKENDS", "onnx libtorch") VALIDATION_FNS = { "onnx": tu.validate_for_onnx_model, - "graphdef": tu.validate_for_tf_model, - "savedmodel": tu.validate_for_tf_model, "libtorch": tu.validate_for_libtorch_model, } diff --git a/qa/L0_inferentia_perf_analyzer/test.sh b/qa/L0_inferentia_perf_analyzer/test.sh deleted file mode 100755 index 1881e07f87..0000000000 --- a/qa/L0_inferentia_perf_analyzer/test.sh +++ /dev/null @@ -1,220 +0,0 @@ -#!/bin/bash -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# First need to set up environment -if [ ${USE_TENSORFLOW} == "1" ] && [ ${USE_PYTORCH} == "1" ] ; then - echo " Unsupported test configuration. Only one of USE_TENSORFLOW and USE_PYTORCH can be set to 1." - exit 0 -elif [ ${USE_TENSORFLOW} == "1" ] ; then - echo "Setting up environment with tensorflow 1" - source ${TRITON_PATH}/python_backend/inferentia/scripts/setup.sh -t --tensorflow-version 1 -elif [ ${USE_PYTORCH} == "1" ] ; then - echo "Setting up environment with pytorch" - source ${TRITON_PATH}/python_backend/inferentia/scripts/setup.sh -p -else - echo " Unsupported test configuration. USE_TENSORFLOW flag is: ${USE_TENSORFLOW} and USE_PYTORCH flag is: ${USE_PYTORCH}. Only one of them can be set to 1." - exit 0 -fi -echo "done setting up environment" - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi - -CLIENT_LOG="./perf_analyzer.log" -PERF_ANALYZER=/opt/tritonserver/qa/clients/perf_analyzer - -OUTPUT_NO_BATCH_JSONDATAFILE=${TEST_JSON_REPO}/validation_no_batch.json -OUTPUT_BATCHED_JSONDATAFILE=${TEST_JSON_REPO}/validation_batched.json -NON_ALIGNED_OUTPUT_NO_BATCH_JSONDATAFILE=${TEST_JSON_REPO}/non_aligned_validation_no_batch.json -NON_ALIGNED_OUTPUT_BATCHED_JSONDATAFILE=${TEST_JSON_REPO}/non_aligned_validation_batched.json -WRONG_OUTPUT_NO_BATCH_JSONDATAFILE=${TEST_JSON_REPO}/wrong_validation_no_batch.json -WRONG_OUTPUT_BATCHED_JSONDATAFILE=${TEST_JSON_REPO}/wrong_validation_batched.json - -ERROR_STRING="error | Request count: 0 | : 0 infer/sec" - -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_LOG="./inference_server.log" -source /opt/tritonserver/qa/common/util.sh -TEST_TYPES="single multiple" -BATCHED_FLAGS="_ _batched_" -DISABLE_DEFAULT_BATCHING_FLAGS="_default_batch _no_batch" -# Helper function for clearing out existing model directories -function clear_model_dir () { - for DISABLE_DEFAULT_BATCHING_FLAG in ${DISABLE_DEFAULT_BATCHING_FLAGS}; do - for BATCHED_FLAG in ${BATCHED_FLAGS}; do - for TEST_TYPE in ${TEST_TYPES}; do - DATADIR="${TRITON_PATH}/models_${TEST_TYPE}${BATCHED_FLAG}${TEST_FRAMEWORK}${DISABLE_DEFAULT_BATCHING_FLAG}" - rm -rf DATADIR - done - done - done -} -# Helper function for generating models -function create_inferentia_models () { - for DISABLE_DEFAULT_BATCHING_FLAG in ${DISABLE_DEFAULT_BATCHING_FLAGS}; do - for BATCHED_FLAG in ${BATCHED_FLAGS}; do - for TEST_TYPE in ${TEST_TYPES}; do - CURR_GEN_SCRIPT="${GEN_SCRIPT} --model_type ${MODEL_TYPE} - --triton_model_dir ${TRITON_PATH}/models_${TEST_TYPE}${BATCHED_FLAG}${TEST_FRAMEWORK}${DISABLE_DEFAULT_BATCHING_FLAG}/add-sub-1x4 - --compiled_model ${COMPILED_MODEL}" - if [ ${DISABLE_DEFAULT_BATCHING_FLAG} == "_no_batch" ]; then - CURR_GEN_SCRIPT="${CURR_GEN_SCRIPT} - --disable_batch_requests_to_neuron" - fi - if [ ${BATCHED_FLAG} == "_batched_" ]; then - CURR_GEN_SCRIPT="${CURR_GEN_SCRIPT} - --triton_input INPUT__0,INT64,4 INPUT__1,INT64,4 - --triton_output OUTPUT__0,INT64,4 OUTPUT__1,INT64,4 - --enable_dynamic_batching - --max_batch_size 1000 - --preferred_batch_size 8 - --max_queue_delay_microseconds 100" - else - CURR_GEN_SCRIPT="${CURR_GEN_SCRIPT} - --triton_input INPUT__0,INT64,-1x4 INPUT__1,INT64,-1x4 - --triton_output OUTPUT__0,INT64,-1x4 OUTPUT__1,INT64,-1x4" - fi - if [ ${TEST_TYPE} == "single" ]; then - CURR_GEN_SCRIPT="${CURR_GEN_SCRIPT} - --neuron_core_range 0:0" - elif [ ${TEST_TYPE} == "multiple" ]; then - CURR_GEN_SCRIPT="${CURR_GEN_SCRIPT} - --triton_model_instance_count 3 - --neuron_core_range 0:7" - fi - echo ${CURR_GEN_SCRIPT} - eval ${CURR_GEN_SCRIPT} - done - done - done -} - -# Setup models -if [ ${USE_TENSORFLOW} == "1" ]; then - TEST_FRAMEWORK="tf1" - clear_model_dir - python ${TEST_JSON_REPO}/simple_model.py \ - --name add_sub_model_tf1 \ - --model_type tensorflow \ - --tf_version 1 \ - --batch_size 1 - GEN_SCRIPT="python ${TRITON_PATH}/python_backend/inferentia/scripts/gen_triton_model.py" - MODEL_TYPE="tensorflow" - COMPILED_MODEL="${PWD}/add_sub_model_tf1" - create_inferentia_models - -elif [ ${USE_PYTORCH} == "1" ]; then - TEST_FRAMEWORK="pyt" - clear_model_dir - python ${TEST_JSON_REPO}/simple_model.py \ - --name add_sub_model_pyt \ - --model_type pytorch \ - --batch_size 1 - GEN_SCRIPT="python ${TRITON_PATH}/python_backend/inferentia/scripts/gen_triton_model.py" - MODEL_TYPE="pytorch" - COMPILED_MODEL="$PWD/add_sub_model_pyt.pt" - create_inferentia_models -fi - - -RET=0 -for DISABLE_DEFAULT_BATCHING_FLAG in ${DISABLE_DEFAULT_BATCHING_FLAGS}; do - for BATCHED_FLAG in ${BATCHED_FLAGS}; do - for TEST_TYPE in $TEST_TYPES; do - DATADIR="${TRITON_PATH}/models_${TEST_TYPE}${BATCHED_FLAG}${TEST_FRAMEWORK}${DISABLE_DEFAULT_BATCHING_FLAG}" - SERVER_ARGS="--model-repository=${DATADIR} --log-verbose=1" - PERF_ANALYZER_EXTRA_ARGS="" - if [ ${BATCHED_FLAG} == "_batched_" ]; then - PERF_ANALYZER_EXTRA_ARGS="-b 6" - NON_ALIGNED_OUTPUT_JSONDATAFILE=${NON_ALIGNED_OUTPUT_BATCHED_JSONDATAFILE} - WRONG_OUTPUT_JSONDATAFILE=${WRONG_OUTPUT_BATCHED_JSONDATAFILE} - OUTPUT_JSONDATAFILE=${OUTPUT_BATCHED_JSONDATAFILE} - else - PERF_ANALYZER_EXTRA_ARGS="" - NON_ALIGNED_OUTPUT_JSONDATAFILE=${NON_ALIGNED_OUTPUT_NO_BATCH_JSONDATAFILE} - WRONG_OUTPUT_JSONDATAFILE=${WRONG_OUTPUT_NO_BATCH_JSONDATAFILE} - OUTPUT_JSONDATAFILE=${OUTPUT_NO_BATCH_JSONDATAFILE} - fi - rm -f $SERVER_LOG $CLIENT_LOG - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - set +e - $PERF_ANALYZER -v -m add-sub-1x4 --concurrency-range 1:10:4 --input-data=${NON_ALIGNED_OUTPUT_JSONDATAFILE} ${PERF_ANALYZER_EXTRA_ARGS} >$CLIENT_LOG 2>&1 - if [ $? -eq 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - if [ $(cat $CLIENT_LOG | grep "The 'validation_data' field doesn't align with 'data' field in the json file" | wc -l) -eq 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - - $PERF_ANALYZER -v -m add-sub-1x4 --concurrency-range 1:10:4 --input-data=${WRONG_OUTPUT_JSONDATAFILE} ${PERF_ANALYZER_EXTRA_ARGS} >$CLIENT_LOG 2>&1 - if [ $? -eq 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - if [ $(cat $CLIENT_LOG | grep "Output doesn't match expected output" | wc -l) -eq 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - - $PERF_ANALYZER -v -m add-sub-1x4 --concurrency-range 1:10:4 --input-data=${OUTPUT_JSONDATAFILE} ${PERF_ANALYZER_EXTRA_ARGS} >$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - if [ $(cat $CLIENT_LOG | grep "${ERROR_STRING}" | wc -l) -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 - fi - set -e - kill_server - done - done -done - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET diff --git a/qa/L0_input_validation/input_validation_test.py b/qa/L0_input_validation/input_validation_test.py index c65c9b6c0c..e3524247aa 100755 --- a/qa/L0_input_validation/input_validation_test.py +++ b/qa/L0_input_validation/input_validation_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -154,7 +154,7 @@ def test_input_shape_validation(self): def test_input_string_shape_validation(self): input_size = 16 - model_name = "graphdef_object_int32_int32" + model_name = "onnx_object_int32_int32" np_dtype_string = np.dtype(object) triton_client = tritongrpcclient.InferenceServerClient("localhost:8001") diff --git a/qa/L0_input_validation/test.sh b/qa/L0_input_validation/test.sh index 22e0560959..2d2cf515ce 100755 --- a/qa/L0_input_validation/test.sh +++ b/qa/L0_input_validation/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -124,7 +124,7 @@ dynamic_batching { } EOL -cp -r $DATADIR/qa_model_repository/graphdef_object_int32_int32 models/. +cp -r $DATADIR/qa_model_repository/onnx_object_int32_int32 models/. cp -r $DATADIR/qa_shapetensor_model_repository/plan_nobatch_zero_1_float32_int32 models/. cp -r $DATADIR/qa_shapetensor_model_repository/plan_zero_1_float32_int32 models/. @@ -151,7 +151,7 @@ kill $SERVER_PID wait $SERVER_PID # input_byte_size_test -cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/{savedmodel_zero_1_float32,savedmodel_zero_1_object} ./models +cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/{onnx_zero_1_float32,onnx_zero_1_object} ./models set +e LD_LIBRARY_PATH=/opt/tritonserver/lib:$LD_LIBRARY_PATH $TEST_EXEC >> $TEST_LOG 2>&1 diff --git a/qa/L0_io/test.sh b/qa/L0_io/test.sh index 0207ce1696..10318da4db 100755 --- a/qa/L0_io/test.sh +++ b/qa/L0_io/test.sh @@ -61,7 +61,7 @@ RET=0 # Prepare float32 models with basic config rm -rf $MODELSDIR -for trial in graphdef savedmodel onnx libtorch plan python python_dlpack; do +for trial in onnx libtorch plan python python_dlpack; do full=${trial}_float32_float32_float32 if [ "$trial" == "python" ]; then mkdir -p $MODELSDIR/${full}/1 && \ @@ -127,7 +127,7 @@ for trial in graphdef savedmodel onnx libtorch plan python python_dlpack; do done # Prepare string models with basic config -for trial in graphdef savedmodel onnx ; do +for trial in onnx ; do full=${trial}_object_object_object mkdir -p $MODELSDIR/${full}/1 && \ cp -r $DATADIR/${full}/1/* $MODELSDIR/${full}/1/. && \ @@ -138,9 +138,9 @@ for trial in graphdef savedmodel onnx ; do done # set up "addsub" ensemble for custom float32 model -cp -r $MODELSDIR/fan_graphdef_float32_float32_float32 $MODELSDIR/fan_${full} && \ +cp -r $MODELSDIR/fan_onnx_float32_float32_float32 $MODELSDIR/fan_${full} && \ (cd $MODELSDIR/fan_${full} && \ - sed -i "s/graphdef_float32_float32_float32/${full}/" config.pbtxt) + sed -i "s/onnx_float32_float32_float32/${full}/" config.pbtxt) # custom float32 component of ensemble cp -r $ENSEMBLEDIR/nop_TYPE_FP32_-1 $MODELSDIR/. && \ @@ -163,7 +163,7 @@ if [ $? -ne 0 ]; then fi set -e -TRIALS="graphdef savedmodel onnx libtorch plan python python_dlpack libtorch_multi_gpu libtorch_multi_device" +TRIALS="onnx libtorch plan python python_dlpack libtorch_multi_gpu libtorch_multi_device" for input_device in -1 0 1; do for output_device in -1 0 1; do for trial in ${TRIALS}; do @@ -230,7 +230,7 @@ for input_device in -1 0 1; do done done - for trial in graphdef savedmodel onnx; do + for trial in onnx; do model_devices="-1 0 1" for model_device in $model_devices; do full=${trial}_object_object_object diff --git a/qa/L0_java_resnet/ResnetTest.java b/qa/L0_java_resnet/ResnetTest.java index 37a8906930..95cf16c6f0 100644 --- a/qa/L0_java_resnet/ResnetTest.java +++ b/qa/L0_java_resnet/ResnetTest.java @@ -37,9 +37,8 @@ public class ResnetTest { // Maximum allowed difference from expected model outputs private static final float ALLOWED_DELTA = .001f; private static final String[] MODELS = { - "resnet50_fp32_libtorch", "resnet50_fp32_onnx", - // TODO: fix build to support GPU only resnet50v1.5_fp16_savedmodel - //"resnet50v1.5_fp16_savedmodel", + "resnet50_fp32_libtorch", + "resnet50_fp32_onnx", }; private static final double TRITON_MIN_COMPUTE_CAPABILITY = 7.5; private enum Backend { @@ -214,7 +213,7 @@ static void GenerateInputData(FloatPointer[] input_data) static boolean AreValidResults( String model_name, FloatPointer output, FloatPointer expected_output) { - int output_length = model_name.contains("tensorflow") ? 1001 : 1000; + int output_length = 1000; for (int i = 0; i < output_length; ++i) { float difference = output.get(i) - expected_output.get(i); if (difference > ALLOWED_DELTA) { @@ -307,9 +306,6 @@ static void Check( case ONNX: file_name += "onnx"; break; - case TF: - file_name += "tensorflow"; - break; case TORCH: file_name += "pytorch"; break; @@ -345,13 +341,11 @@ static void PerformInference( Backend backend = Backend.NONE; if (model_name.contains("onnx")) { backend = Backend.ONNX; - } else if (model_name.contains("savedmodel")) { - backend = Backend.TF; } else if (model_name.contains("torch")) { backend = Backend.TORCH; } else { FAIL( - "Supported model types (Onnx, TensorFlow, Torch) " + "Supported model types (Onnx, Torch) " + "cannot be inferred from model name " + model_name); } diff --git a/qa/L0_java_resnet/test.sh b/qa/L0_java_resnet/test.sh index 11eeeeb114..8b7e6b8a07 100755 --- a/qa/L0_java_resnet/test.sh +++ b/qa/L0_java_resnet/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ JAVACPP_BRANCH_TAG=${JAVACPP_BRANCH_TAG:="master"} # Create local model repository mkdir -p ${MODEL_REPO} -# TODO: fix build to support GPU only resnet50v1.5_fp16_savedmodel for BACKEND in _fp32_libtorch _fp32_onnx; do cp -r $DATADIR/perf_model_store/resnet50${BACKEND} ${MODEL_REPO}/ echo ${MODEL_REPO}/resnet50${BACKEND}/config.pbtxt diff --git a/qa/L0_java_sequence_batcher/SequenceTest.java b/qa/L0_java_sequence_batcher/SequenceTest.java index cfce3584de..484906f91d 100644 --- a/qa/L0_java_sequence_batcher/SequenceTest.java +++ b/qa/L0_java_sequence_batcher/SequenceTest.java @@ -251,8 +251,8 @@ static int GetExpectedResult( String model_name, int expected_result, int value, String flag) { if ((!model_name.contains("nobatch") && !model_name.contains("custom")) - || model_name.contains("graphdef") || model_name.contains("plan") - || model_name.contains("onnx") || model_name.contains("libtorch")) { + || model_name.contains("plan") || model_name.contains("onnx") + || model_name.contains("libtorch")) { expected_result = value; if (flag != null && flag.contains("start")) { expected_result++; diff --git a/qa/L0_java_sequence_batcher/test.sh b/qa/L0_java_sequence_batcher/test.sh index aadf3172be..5c0781bcda 100755 --- a/qa/L0_java_sequence_batcher/test.sh +++ b/qa/L0_java_sequence_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -63,7 +63,7 @@ sed -i 's/Simple/SequenceTest/g' $SAMPLES_REPO/pom.xml rm -f *.log RET=0 -for BACKEND in graphdef libtorch onnx savedmodel; do +for BACKEND in libtorch onnx; do # Create local model repository mkdir -p ${MODEL_REPO} MODEL=${BACKEND}_nobatch_sequence_int32 diff --git a/qa/L0_java_simple_example/test.sh b/qa/L0_java_simple_example/test.sh index 3b3fa41f0b..0441e88641 100755 --- a/qa/L0_java_simple_example/test.sh +++ b/qa/L0_java_simple_example/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -104,7 +104,7 @@ function run_cpu_tests_int32() { } function run_cpu_tests_fp32() { - for trial in graphdef savedmodel; do + for trial in onnx; do full=${trial}_float32_float32_float32 set +e rm -rf ${MODEL_REPO} diff --git a/qa/L0_large_payload/large_payload_test.py b/qa/L0_large_payload/large_payload_test.py index f9c0a49dfd..1ab6d6e340 100755 --- a/qa/L0_large_payload/large_payload_test.py +++ b/qa/L0_large_payload/large_payload_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -134,20 +134,6 @@ def _test_helper( "output is different from input", ) - def test_graphdef(self): - # graphdef_nobatch_zero_1_float32 is identity model with input shape [-1] - for client in self._clients: - model_name = tu.get_zero_model_name("graphdef_nobatch", 1, self._data_type) - self._test_helper(client, model_name) - - def test_savedmodel(self): - # savedmodel_nobatch_zero_1_float32 is identity model with input shape [-1] - for client in self._clients: - model_name = tu.get_zero_model_name( - "savedmodel_nobatch", 1, self._data_type - ) - self._test_helper(client, model_name) - def test_onnx(self): # onnx_nobatch_zero_1_float32 is identity model with input shape [-1] for client in self._clients: diff --git a/qa/L0_large_payload/test.sh b/qa/L0_large_payload/test.sh index 325cab4ed5..b4aff503d1 100755 --- a/qa/L0_large_payload/test.sh +++ b/qa/L0_large_payload/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -56,7 +56,7 @@ RET=0 MODEL_SUFFIX=nobatch_zero_1_float32 rm -fr all_models && mkdir all_models -for TARGET in graphdef savedmodel onnx libtorch plan; do +for TARGET in onnx libtorch plan; do cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/${TARGET}_$MODEL_SUFFIX \ all_models/. done @@ -71,7 +71,7 @@ cp ../python_models/identity_fp32/model.py all_models/python_$MODEL_SUFFIX/1/mod # Restart server before every test to make sure server state # is invariant to previous test -for TARGET in graphdef savedmodel onnx libtorch plan python; do +for TARGET in onnx libtorch plan python; do rm -fr models && mkdir models && \ cp -r all_models/${TARGET}_$MODEL_SUFFIX models/. diff --git a/qa/L0_lifecycle/lifecycle_test.py b/qa/L0_lifecycle/lifecycle_test.py index 1dc659e27c..8ad95e3d90 100755 --- a/qa/L0_lifecycle/lifecycle_test.py +++ b/qa/L0_lifecycle/lifecycle_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -172,7 +172,7 @@ def test_parse_error_modelfail(self): # Server was started but with a model that fails to load try: model_name = tu.get_model_name( - "graphdef", np.float32, np.float32, np.float32 + "libtorch", np.float32, np.float32, np.float32 ) triton_client = grpcclient.InferenceServerClient( @@ -194,18 +194,18 @@ def test_parse_error_modelfail(self): # Inferencing with the missing model should fail. try: iu.infer_exact( - self, "graphdef", tensor_shape, 1, np.float32, np.float32, np.float32 + self, "libtorch", tensor_shape, 1, np.float32, np.float32, np.float32 ) self.assertTrue(False, "expected error for unavailable model " + model_name) except Exception as ex: self.assertIn( - "Request for unknown model: 'graphdef_float32_float32_float32' has no available versions", + "Request for unknown model: 'libtorch_float32_float32_float32' has no available versions", ex.message(), ) # And other models should be loaded successfully try: - for base_name in ["savedmodel", "onnx"]: + for base_name in ["openvino", "onnx"]: for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), grpcclient.InferenceServerClient("localhost:8001", verbose=True), @@ -235,7 +235,7 @@ def test_parse_error_modelfail_nostrict(self): # Server was started but with a model that fails to load try: model_name = tu.get_model_name( - "graphdef", np.float32, np.float32, np.float32 + "libtorch", np.float32, np.float32, np.float32 ) triton_client = grpcclient.InferenceServerClient( @@ -257,18 +257,18 @@ def test_parse_error_modelfail_nostrict(self): # Inferencing with the missing model should fail. try: iu.infer_exact( - self, "graphdef", tensor_shape, 1, np.float32, np.float32, np.float32 + self, "libtorch", tensor_shape, 1, np.float32, np.float32, np.float32 ) self.assertTrue(False, "expected error for unavailable model " + model_name) except Exception as ex: self.assertIn( - "Request for unknown model: 'graphdef_float32_float32_float32' has no available versions", + "Request for unknown model: 'libtorch_float32_float32_float32' has no available versions", ex.message(), ) # And other models should be loaded successfully try: - for base_name in ["savedmodel", "onnx"]: + for base_name in ["openvino", "onnx"]: for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), grpcclient.InferenceServerClient("localhost:8001", verbose=True), @@ -301,7 +301,7 @@ def test_parse_error_no_model_config(self): ): try: model_name = tu.get_model_name( - "graphdef", np.float32, np.float32, np.float32 + "openvino", np.float32, np.float32, np.float32 ) # expecting ready because not strict readiness @@ -318,13 +318,13 @@ def test_parse_error_no_model_config(self): except Exception as ex: self.assertIn( - "Request for unknown model: 'graphdef_float32_float32_float32' is not found", + "Request for unknown model: 'openvino_float32_float32_float32' is not found", ex.message(), ) # And other models should be loaded successfully try: - for base_name in ["savedmodel", "onnx"]: + for base_name in ["libtorch", "onnx"]: model_name = tu.get_model_name( base_name, np.float32, np.float32, np.float32 ) @@ -365,7 +365,7 @@ def test_init_error_modelfail(self): # And other models should be loaded successfully try: - for base_name in ["graphdef", "savedmodel", "onnx"]: + for base_name in ["openvino", "libtorch", "onnx"]: model_name = tu.get_model_name( base_name, np.float32, np.float32, np.float32 ) @@ -375,7 +375,7 @@ def test_init_error_modelfail(self): try: tensor_shape = (1, 16) - for base_name in ["graphdef", "savedmodel", "onnx"]: + for base_name in ["openvino", "libtorch", "onnx"]: iu.infer_exact( self, base_name, @@ -403,7 +403,7 @@ def test_parse_error_model_no_version(self): self.assertFalse(triton_client.is_server_ready()) model_name = tu.get_model_name( - "graphdef", np.float32, np.float32, np.float32 + "openvino", np.float32, np.float32, np.float32 ) self.assertFalse(triton_client.is_model_ready(model_name)) except Exception as ex: @@ -411,7 +411,7 @@ def test_parse_error_model_no_version(self): # Sanity check that other models are loaded properly try: - for base_name in ["savedmodel", "onnx"]: + for base_name in ["libtorch", "onnx"]: model_name = tu.get_model_name( base_name, np.float32, np.float32, np.float32 ) @@ -425,7 +425,7 @@ def test_parse_error_model_no_version(self): self.assertTrue(False, "unexpected error {}".format(ex)) try: - for base_name in ["savedmodel", "onnx"]: + for base_name in ["libtorch", "onnx"]: iu.infer_exact( self, base_name, @@ -453,12 +453,12 @@ def test_parse_error_model_no_version(self): try: iu.infer_exact( - self, "graphdef", tensor_shape, 1, np.float32, np.float32, np.float32 + self, "openvino", tensor_shape, 1, np.float32, np.float32, np.float32 ) self.assertTrue(False, "expected error for unavailable model " + model_name) except Exception as ex: self.assertIn( - "Request for unknown model: 'graphdef_float32_float32_float32' has no available versions", + "Request for unknown model: 'openvino_float32_float32_float32' has no available versions", ex.message(), ) @@ -475,7 +475,7 @@ def test_parse_ignore_zero_prefixed_version(self): self.assertTrue(triton_client.is_server_ready()) model_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + "libtorch", np.float32, np.float32, np.float32 ) self.assertTrue(triton_client.is_model_ready(model_name, "1")) except Exception as ex: @@ -485,7 +485,7 @@ def test_parse_ignore_zero_prefixed_version(self): # swap=False for version 1 iu.infer_exact( self, - "savedmodel", + "libtorch", tensor_shape, 1, np.float32, @@ -509,7 +509,7 @@ def test_parse_ignore_non_intergral_version(self): self.assertTrue(triton_client.is_server_ready()) model_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + "libtorch", np.float32, np.float32, np.float32 ) self.assertTrue(triton_client.is_model_ready(model_name, "1")) except Exception as ex: @@ -519,7 +519,7 @@ def test_parse_ignore_non_intergral_version(self): # swap=False for version 1 iu.infer_exact( self, - "savedmodel", + "libtorch", tensor_shape, 1, np.float32, @@ -532,12 +532,12 @@ def test_parse_ignore_non_intergral_version(self): def test_dynamic_model_load_unload(self): tensor_shape = (1, 16) - savedmodel_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + libtorch_name = tu.get_model_name( + "libtorch", np.float32, np.float32, np.float32 ) onnx_name = tu.get_model_name("onnx", np.float32, np.float32, np.float32) - # Make sure savedmodel model is not in the status (because + # Make sure libtorch model is not in the status (because # initially it is not in the model repository) for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -546,17 +546,17 @@ def test_dynamic_model_load_unload(self): try: self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - # Add savedmodel model to the model repository and give it time to + # Add libtorch model to the model repository and give it time to # load. Make sure that it has a status and is ready. try: - shutil.copytree(savedmodel_name, "models/" + savedmodel_name) + shutil.copytree(libtorch_name, "models/" + libtorch_name) time.sleep(5) # wait for model to load for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -564,8 +564,8 @@ def test_dynamic_model_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: @@ -575,7 +575,7 @@ def test_dynamic_model_load_unload(self): try: iu.infer_exact( self, - "savedmodel", + "libtorch", tensor_shape, 1, np.float32, @@ -586,15 +586,15 @@ def test_dynamic_model_load_unload(self): except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - # Make sure savedmodel has execution stats + # Make sure libtorch has execution stats try: triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True ) - stats = triton_client.get_inference_statistics(savedmodel_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats["model_stats"]), 2) for idx in range(len(stats["model_stats"])): - self.assertEqual(stats["model_stats"][idx]["name"], savedmodel_name) + self.assertEqual(stats["model_stats"][idx]["name"], libtorch_name) if stats["model_stats"][idx]["version"] == "1": self.assertEqual( stats["model_stats"][idx]["inference_stats"]["success"][ @@ -613,10 +613,10 @@ def test_dynamic_model_load_unload(self): triton_client = grpcclient.InferenceServerClient( "localhost:8001", verbose=True ) - stats = triton_client.get_inference_statistics(savedmodel_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats.model_stats), 2) for idx in range(len(stats.model_stats)): - self.assertEqual(stats.model_stats[idx].name, savedmodel_name) + self.assertEqual(stats.model_stats[idx].name, libtorch_name) if stats.model_stats[idx].version == "1": self.assertEqual( stats.model_stats[idx].inference_stats.success.count, 0 @@ -629,10 +629,10 @@ def test_dynamic_model_load_unload(self): except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - # Remove savedmodel model from the model repository and give it + # Remove libtorch model from the model repository and give it # time to unload. Make sure that it is no longer available. try: - shutil.rmtree("models/" + savedmodel_name) + shutil.rmtree("models/" + libtorch_name) time.sleep(5) # wait for model to unload for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -640,8 +640,8 @@ def test_dynamic_model_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: @@ -651,7 +651,7 @@ def test_dynamic_model_load_unload(self): try: iu.infer_exact( self, - "savedmodel", + "libtorch", tensor_shape, 1, np.float32, @@ -660,19 +660,19 @@ def test_dynamic_model_load_unload(self): swap=True, ) self.assertTrue( - False, "expected error for unavailable model " + savedmodel_name + False, "expected error for unavailable model " + libtorch_name ) except Exception as ex: self.assertIn( "Request for unknown model: '{}' has no available versions".format( - savedmodel_name + libtorch_name ), ex.message(), ) # Add back the same model. The status/stats should be reset. try: - shutil.copytree(savedmodel_name, "models/" + savedmodel_name) + shutil.copytree(libtorch_name, "models/" + libtorch_name) time.sleep(5) # wait for model to load for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -680,18 +680,18 @@ def test_dynamic_model_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True ) - stats = triton_client.get_inference_statistics(savedmodel_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats["model_stats"]), 2) - self.assertEqual(stats["model_stats"][0]["name"], savedmodel_name) - self.assertEqual(stats["model_stats"][1]["name"], savedmodel_name) + self.assertEqual(stats["model_stats"][0]["name"], libtorch_name) + self.assertEqual(stats["model_stats"][1]["name"], libtorch_name) self.assertEqual( stats["model_stats"][0]["inference_stats"]["success"]["count"], 0 ) @@ -702,10 +702,10 @@ def test_dynamic_model_load_unload(self): triton_client = grpcclient.InferenceServerClient( "localhost:8001", verbose=True ) - stats = triton_client.get_inference_statistics(savedmodel_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats.model_stats), 2) - self.assertEqual(stats.model_stats[0].name, savedmodel_name) - self.assertEqual(stats.model_stats[1].name, savedmodel_name) + self.assertEqual(stats.model_stats[0].name, libtorch_name) + self.assertEqual(stats.model_stats[1].name, libtorch_name) self.assertEqual(stats.model_stats[0].inference_stats.success.count, 0) self.assertEqual(stats.model_stats[1].inference_stats.success.count, 0) @@ -723,8 +723,8 @@ def test_dynamic_model_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) self.assertFalse(triton_client.is_model_ready(onnx_name, "1")) self.assertFalse(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: @@ -751,12 +751,12 @@ def test_dynamic_model_load_unload(self): def test_dynamic_model_load_unload_disabled(self): tensor_shape = (1, 16) - savedmodel_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + libtorch_name = tu.get_model_name( + "libtorch", np.float32, np.float32, np.float32 ) onnx_name = tu.get_model_name("onnx", np.float32, np.float32, np.float32) - # Make sure savedmodel model is not in the status (because + # Make sure libtorch model is not in the status (because # initially it is not in the model repository) for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -765,17 +765,17 @@ def test_dynamic_model_load_unload_disabled(self): try: self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - # Add savedmodel model to the model repository and give it time to + # Add libtorch model to the model repository and give it time to # load. But it shouldn't load because dynamic loading is disabled. try: - shutil.copytree(savedmodel_name, "models/" + savedmodel_name) + shutil.copytree(libtorch_name, "models/" + libtorch_name) time.sleep(5) # wait for model to load for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -783,8 +783,8 @@ def test_dynamic_model_load_unload_disabled(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: @@ -794,7 +794,7 @@ def test_dynamic_model_load_unload_disabled(self): try: iu.infer_exact( self, - "savedmodel", + "libtorch", tensor_shape, 1, np.float32, @@ -803,11 +803,11 @@ def test_dynamic_model_load_unload_disabled(self): swap=True, ) self.assertTrue( - False, "expected error for unavailable model " + savedmodel_name + False, "expected error for unavailable model " + libtorch_name ) except Exception as ex: self.assertIn( - "Request for unknown model: 'savedmodel_float32_float32_float32' is not found", + "Request for unknown model: 'libtorch_float32_float32_float32' is not found", ex.message(), ) @@ -822,8 +822,8 @@ def test_dynamic_model_load_unload_disabled(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) self.assertTrue(triton_client.is_model_ready(onnx_name, "1")) self.assertTrue(triton_client.is_model_ready(onnx_name, "3")) except Exception as ex: @@ -847,7 +847,7 @@ def test_dynamic_model_load_unload_disabled(self): def test_dynamic_version_load_unload(self): tensor_shape = (1, 16) - graphdef_name = tu.get_model_name("graphdef", np.int32, np.int32, np.int32) + libtorch_name = tu.get_model_name("libtorch", np.int32, np.int32, np.int32) # There are 3 versions. Make sure that all have status and are # ready. @@ -858,9 +858,9 @@ def test_dynamic_version_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "1")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "2")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "3")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "2")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) @@ -868,7 +868,7 @@ def test_dynamic_version_load_unload(self): try: iu.infer_exact( self, - "graphdef", + "libtorch", tensor_shape, 1, np.int32, @@ -885,10 +885,10 @@ def test_dynamic_version_load_unload(self): triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True ) - stats = triton_client.get_inference_statistics(graphdef_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats["model_stats"]), 3) for idx in range(len(stats["model_stats"])): - self.assertEqual(stats["model_stats"][idx]["name"], graphdef_name) + self.assertEqual(stats["model_stats"][idx]["name"], libtorch_name) if stats["model_stats"][idx]["version"] == "1": self.assertNotEqual( stats["model_stats"][idx]["inference_stats"]["success"][ @@ -907,10 +907,10 @@ def test_dynamic_version_load_unload(self): triton_client = grpcclient.InferenceServerClient( "localhost:8001", verbose=True ) - stats = triton_client.get_inference_statistics(graphdef_name) + stats = triton_client.get_inference_statistics(libtorch_name) self.assertEqual(len(stats.model_stats), 3) for idx in range(len(stats.model_stats)): - self.assertEqual(stats.model_stats[idx].name, graphdef_name) + self.assertEqual(stats.model_stats[idx].name, libtorch_name) if stats.model_stats[idx].version == "1": self.assertNotEqual( stats.model_stats[idx].inference_stats.success.count, 0 @@ -926,7 +926,7 @@ def test_dynamic_version_load_unload(self): # Remove version 1 from the model repository and give it time to # unload. Make sure that it is unavailable. try: - shutil.rmtree("models/" + graphdef_name + "/1") + shutil.rmtree("models/" + libtorch_name + "/1") time.sleep(5) # wait for version to unload for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -934,9 +934,9 @@ def test_dynamic_version_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(graphdef_name, "1")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "2")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "2")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) @@ -944,7 +944,7 @@ def test_dynamic_version_load_unload(self): try: iu.infer_exact( self, - "graphdef", + "libtorch", tensor_shape, 1, np.int32, @@ -954,18 +954,18 @@ def test_dynamic_version_load_unload(self): model_version=1, ) self.assertTrue( - False, "expected error for unavailable model " + graphdef_name + False, "expected error for unavailable model " + libtorch_name ) except Exception as ex: self.assertIn( - "Request for unknown model: 'graphdef_int32_int32_int32' version 1 is not at ready state", + "Request for unknown model: 'libtorch_int32_int32_int32' version 1 is not at ready state", ex.message(), ) # Add another version to the model repository. try: shutil.copytree( - "models/" + graphdef_name + "/2", "models/" + graphdef_name + "/7" + "models/" + libtorch_name + "/2", "models/" + libtorch_name + "/7" ) time.sleep(5) # wait for version to load for triton_client in ( @@ -974,23 +974,23 @@ def test_dynamic_version_load_unload(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(graphdef_name, "1")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "2")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "3")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "7")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "2")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "7")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) def test_dynamic_version_load_unload_disabled(self): tensor_shape = (1, 16) - graphdef_name = tu.get_model_name("graphdef", np.int32, np.int32, np.int32) + libtorch_name = tu.get_model_name("libtorch", np.int32, np.int32, np.int32) # Add a new version to the model repository and give it time to # load. But it shouldn't load because dynamic loading is # disabled. try: shutil.copytree( - "models/" + graphdef_name + "/2", "models/" + graphdef_name + "/7" + "models/" + libtorch_name + "/2", "models/" + libtorch_name + "/7" ) time.sleep(5) # wait for model to load for triton_client in ( @@ -999,10 +999,10 @@ def test_dynamic_version_load_unload_disabled(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "1")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "2")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "3")) - self.assertFalse(triton_client.is_model_ready(graphdef_name, "7")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "2")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "7")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) @@ -1010,7 +1010,7 @@ def test_dynamic_version_load_unload_disabled(self): # Unloading is disabled so it should remain available # in the status. try: - shutil.rmtree("models/" + graphdef_name + "/1") + shutil.rmtree("models/" + libtorch_name + "/1") time.sleep(5) # wait for version to unload (but it shouldn't) for triton_client in ( httpclient.InferenceServerClient("localhost:8000", verbose=True), @@ -1018,10 +1018,10 @@ def test_dynamic_version_load_unload_disabled(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "1")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "2")) - self.assertTrue(triton_client.is_model_ready(graphdef_name, "3")) - self.assertFalse(triton_client.is_model_ready(graphdef_name, "7")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "2")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "7")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) @@ -1030,7 +1030,7 @@ def test_dynamic_version_load_unload_disabled(self): try: iu.infer_exact( self, - "graphdef", + "libtorch", tensor_shape, 1, np.int32, @@ -1043,13 +1043,13 @@ def test_dynamic_version_load_unload_disabled(self): self.assertTrue(False, "unexpected error {}".format(ex)) def test_dynamic_model_modify(self): - models_base = ("savedmodel", "plan") + models_base = ("libtorch", "plan") models_shape = ((1, 16), (1, 16)) models = list() for m in models_base: models.append(tu.get_model_name(m, np.float32, np.float32, np.float32)) - # Make sure savedmodel and plan are in the status + # Make sure libtorch and plan are in the status for model_name in models: try: for triton_client in ( @@ -1170,13 +1170,13 @@ def test_dynamic_model_modify(self): self.assertTrue(False, "unexpected error {}".format(ex)) def test_dynamic_file_delete(self): - models_base = ("savedmodel", "plan") + models_base = ("onnx", "plan") models_shape = ((1, 16), (1, 16)) models = list() for m in models_base: models.append(tu.get_model_name(m, np.float32, np.float32, np.float32)) - # Make sure savedmodel and plan are in the status + # Make sure onnx and plan are in the status for model_name in models: try: for triton_client in ( @@ -1266,24 +1266,24 @@ def test_dynamic_file_delete(self): def test_multiple_model_repository_polling(self): model_shape = (1, 16) - savedmodel_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + libtorch_name = tu.get_model_name( + "libtorch", np.float32, np.float32, np.float32 ) # Models should be loaded successfully and infer - # successfully. Initially savedmodel only has version 1. + # successfully. Initially libtorch only has version 1. self._infer_success_models( [ - "savedmodel", + "libtorch", ], (1,), model_shape, ) - self._infer_success_models(["graphdef", "onnx"], (1, 3), model_shape) + self._infer_success_models(["openvino", "onnx"], (1, 3), model_shape) - # Add the savedmodel to the second model repository, should cause + # Add the libtorch to the second model repository, should cause # it to be unloaded due to duplication - shutil.copytree(savedmodel_name, "models_0/" + savedmodel_name) + shutil.copytree(libtorch_name, "models_0/" + libtorch_name) time.sleep(5) # wait for models to reload try: for triton_client in ( @@ -1292,31 +1292,31 @@ def test_multiple_model_repository_polling(self): ): self.assertTrue(triton_client.is_server_live()) self.assertTrue(triton_client.is_server_ready()) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "1")) - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "1")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - self._infer_success_models(["graphdef", "onnx"], (1, 3), model_shape) + self._infer_success_models(["openvino", "onnx"], (1, 3), model_shape) - # Remove the savedmodel from the first model repository, the + # Remove the libtorch from the first model repository, the # model from the second model repository should be loaded - # properly. In the second model repository savedmodel should + # properly. In the second model repository libtorch should # have versions 1 and 3. - shutil.rmtree("models/" + savedmodel_name) + shutil.rmtree("models/" + libtorch_name) time.sleep(5) # wait for model to unload self._infer_success_models( - ["savedmodel", "graphdef", "onnx"], (1, 3), model_shape + ["libtorch", "openvino", "onnx"], (1, 3), model_shape ) def test_multiple_model_repository_control(self): # similar to test_multiple_model_repository_polling, but the # model load/unload is controlled by the API model_shape = (1, 16) - savedmodel_name = tu.get_model_name( - "savedmodel", np.float32, np.float32, np.float32 + libtorch_name = tu.get_model_name( + "libtorch", np.float32, np.float32, np.float32 ) - model_bases = ["savedmodel", "graphdef", "onnx"] + model_bases = ["libtorch", "openvino", "onnx"] # Initially models are not loaded for base in model_bases: @@ -1345,38 +1345,38 @@ def test_multiple_model_repository_control(self): self.assertTrue(False, "unexpected error {}".format(ex)) # Models should be loaded successfully and infer - # successfully. Initially savedmodel only has version 1. + # successfully. Initially libtorch only has version 1. self._infer_success_models( [ - "savedmodel", + "libtorch", ], (1,), model_shape, ) - self._infer_success_models(["graphdef", "onnx"], (1, 3), model_shape) + self._infer_success_models(["openvino", "onnx"], (1, 3), model_shape) - # Add the savedmodel to the second model repository. Because + # Add the libtorch to the second model repository. Because # not polling this doesn't change any model state, all models # are still loaded and available. - shutil.copytree(savedmodel_name, "models_0/" + savedmodel_name) + shutil.copytree(libtorch_name, "models_0/" + libtorch_name) self._infer_success_models( [ - "savedmodel", + "libtorch", ], (1,), model_shape, ) - self._infer_success_models(["graphdef", "onnx"], (1, 3), model_shape) + self._infer_success_models(["openvino", "onnx"], (1, 3), model_shape) - # Load savedmodel again which should fail because it is now duplicated + # Load libtorch again which should fail because it is now duplicated # in 2 model repositories. Use HTTP here. try: triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True ) - triton_client.load_model(savedmodel_name) + triton_client.load_model(libtorch_name) except Exception as ex: - self.assertIn("failed to load '{}'".format(savedmodel_name), ex.message()) + self.assertIn("failed to load '{}'".format(libtorch_name), ex.message()) try: for triton_client in ( @@ -1387,33 +1387,33 @@ def test_multiple_model_repository_control(self): self.assertTrue(triton_client.is_server_ready()) # Unlike polling mode, the failed load on the duplicate model # should NOT unload the existing versions in model control mode. - self.assertTrue(triton_client.is_model_ready(savedmodel_name, "1")) + self.assertTrue(triton_client.is_model_ready(libtorch_name, "1")) # Version 3 did not exist in the first model repository, so # it should still not be loaded. - self.assertFalse(triton_client.is_model_ready(savedmodel_name, "3")) + self.assertFalse(triton_client.is_model_ready(libtorch_name, "3")) except Exception as ex: self.assertTrue(False, "unexpected error {}".format(ex)) - self._infer_success_models(["graphdef", "onnx"], (1, 3), model_shape) + self._infer_success_models(["openvino", "onnx"], (1, 3), model_shape) - # Remove the savedmodel from the first model repository and - # explicitly load savedmodel. The savedmodel from the second + # Remove the libtorch from the first model repository and + # explicitly load libtorch. The libtorch from the second # model repository should be loaded properly. In the second - # model repository savedmodel should have versions 1 and 3. - shutil.rmtree("models/" + savedmodel_name) + # model repository libtorch should have versions 1 and 3. + shutil.rmtree("models/" + libtorch_name) try: triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True ) # Unload existing in-memory model from first model repository - triton_client.unload_model(savedmodel_name) + triton_client.unload_model(libtorch_name) # Load model from second model repository since original was deleted - triton_client.load_model(savedmodel_name) + triton_client.load_model(libtorch_name) except Exception as ex: - self.assertIn("failed to load '{}'".format(savedmodel_name), ex.message()) + self.assertIn("failed to load '{}'".format(libtorch_name), ex.message()) self._infer_success_models( - ["savedmodel", "graphdef", "onnx"], (1, 3), model_shape + ["libtorch", "openvino", "onnx"], (1, 3), model_shape ) def test_model_control(self): @@ -2038,7 +2038,7 @@ def test_multiple_model_repository_control_startup_models(self): plan_ensemble_name = ensemble_prefix + plan_name # Make sure unloaded models are not in the status - for base in ("savedmodel",): + for base in ("libtorch",): model_name = tu.get_model_name(base, np.float32, np.float32, np.float32) try: for triton_client in ( @@ -2257,13 +2257,12 @@ def test_model_repository_index(self): # use model control EXPLICIT and --load-model to load a subset of models # in model repository tensor_shape = (1, 16) - model_bases = ["graphdef", "savedmodel", "simple_savedmodel"] + model_bases = ["plan", "libtorch", "simple_libtorch"] # Sanity check on loaded models - # 3 models should be loaded: - # simple_savedmodel_float32_float32_float32 - # savedmodel_float32_float32_float32 - # graphdef_float32_float32_float32 + # 2 models should be loaded: + # simple_libtorch_float32_float32_float32 + # libtorch_float32_float32_float32 for model_base in model_bases: try: model_name = tu.get_model_name( @@ -2282,7 +2281,7 @@ def test_model_repository_index(self): # Check model repository index # All models should be in ready state except onnx_float32_float32_float32 # which appears in two repositories. - model_bases.append("simple_graphdef") + model_bases.append("simple_plan") try: triton_client = httpclient.InferenceServerClient( "localhost:8000", verbose=True @@ -3406,12 +3405,12 @@ def test_shutdown_with_live_connection(self): ) def test_add_custom_config(self): - models_base = ("savedmodel",) + models_base = ("libtorch",) models = list() for m in models_base: models.append(tu.get_model_name(m, np.float32, np.float32, np.float32)) - # Make sure savedmodel and plan are in the status + # Make sure libtorch and plan are in the status for model_name in models: try: for triton_client in ( @@ -3452,12 +3451,12 @@ def test_add_custom_config(self): self.assertTrue(False, "unexpected error {}".format(ex)) def test_delete_custom_config(self): - models_base = ("savedmodel",) + models_base = ("libtorch",) models = list() for m in models_base: models.append(tu.get_model_name(m, np.float32, np.float32, np.float32)) - # Make sure savedmodel and plan are in the status + # Make sure libtorch and plan are in the status for model_name in models: try: for triton_client in ( diff --git a/qa/L0_lifecycle/test.sh b/qa/L0_lifecycle/test.sh index f1a33886ac..bcd1713bb8 100755 --- a/qa/L0_lifecycle/test.sh +++ b/qa/L0_lifecycle/test.sh @@ -333,15 +333,18 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_parse_error_modelfail rm -fr models models_0 mkdir models models_0 -for i in graphdef savedmodel ; do +for i in openvino libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. + if [ $i == "openvino" ]; then + echo 'parameters { key: "ENABLE_BATCH_PADDING" value { string_value: "YES" } }' >> models/openvino_float32_float32_float32/config.pbtxt + fi done for i in onnx plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done # Change the model files so that multiple versions will be loaded, and one of # the versions will fail to load and cause all other versions to be unloaded. -rm models/graphdef_float32_float32_float32/3/* +rm models/libtorch_float32_float32_float32/3/* SERVER_ARGS="--model-repository=`pwd`/models --model-repository=`pwd`/models_0 \ --exit-on-error=false --exit-timeout-secs=5" @@ -395,13 +398,13 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_parse_error_no_model_config rm -fr models models_0 mkdir models models_0 -for i in graphdef savedmodel ; do +for i in openvino libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done for i in onnx plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done -rm models/graphdef_float32_float32_float32/config.pbtxt +rm models/openvino_float32_float32_float32/config.pbtxt # Autocomplete should not be turned on for this test because it asserts an error was logged # when in strict model configuration mode. @@ -428,7 +431,7 @@ kill $SERVER_PID wait $SERVER_PID # check server log for the warning messages -if [ `grep -c "failed to open text file for read" $SERVER_LOG` == "0" ] || [ `grep -c "graphdef_float32_float32_float32/config.pbtxt: No such file or directory" $SERVER_LOG` == "0" ]; then +if [ `grep -c "failed to open text file for read" $SERVER_LOG` == "0" ] || [ `grep -c "openvino_float32_float32_float32/config.pbtxt: No such file or directory" $SERVER_LOG` == "0" ]; then echo -e "\n***\n*** Server log ${SERVER_LOG} did not print model load failure\n***" echo -e "\n***\n*** Test Failed\n***" RET=1 @@ -443,8 +446,11 @@ cp -r $DATADIR/qa_sequence_model_repository/onnx_sequence_int32 models/. cp -r $DATADIR/qa_model_repository/onnx_int32_int32_int32 models_0/. sed -i "s/OUTPUT/_OUTPUT/" models/onnx_sequence_int32/config.pbtxt sed -i "s/OUTPUT/_OUTPUT/" models_0/onnx_int32_int32_int32/config.pbtxt -for i in graphdef savedmodel; do +for i in openvino libtorch; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. + if [ $i == "openvino" ]; then + echo 'parameters { key: "ENABLE_BATCH_PADDING" value { string_value: "YES" } }' >> models/openvino_float32_float32_float32/config.pbtxt + fi done for i in onnx ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. @@ -477,12 +483,12 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_parse_error_model_no_version rm -fr models mkdir models -for i in savedmodel onnx plan ; do +for i in libtorch onnx plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done -mkdir -p models/graphdef_float32_float32_float32 -cp $DATADIR/qa_model_repository/graphdef_float32_float32_float32/config.pbtxt \ - models/graphdef_float32_float32_float32/. +mkdir -p models/openvino_float32_float32_float32 +cp $DATADIR/qa_model_repository/openvino_float32_float32_float32/config.pbtxt \ + models/openvino_float32_float32_float32/. SERVER_ARGS="--model-repository=`pwd`/models --exit-on-error=false \ --exit-timeout-secs=5" @@ -511,7 +517,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_parse_ignore_zero_prefixed_version rm -fr models mkdir models -for i in savedmodel ; do +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. mv models/${i}_float32_float32_float32/3 models/${i}_float32_float32_float32/003 done @@ -546,7 +552,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_parse_ignore_non_intergral_version rm -fr models mkdir models -for i in savedmodel ; do +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. mv models/${i}_float32_float32_float32/3 models/${i}_float32_float32_float32/abc done @@ -579,12 +585,12 @@ fi LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_model_load_unload -rm -fr models savedmodel_float32_float32_float32 +rm -fr models libtorch_float32_float32_float32 mkdir models -for i in graphdef onnx plan ; do +for i in openvino onnx plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 . +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 . SERVER_ARGS="--model-repository=`pwd`/models --repository-poll-secs=1 \ --model-control-mode=poll --exit-timeout-secs=5" @@ -608,12 +614,12 @@ wait $SERVER_PID LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_model_load_unload_disabled -rm -fr models savedmodel_float32_float32_float32 +rm -fr models libtorch_float32_float32_float32 mkdir models -for i in graphdef onnx plan; do +for i in openvino onnx plan; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 . +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 . SERVER_ARGS="--model-repository=`pwd`/models --model-control-mode=none \ --exit-timeout-secs=5" @@ -639,7 +645,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_version_load_unload rm -fr models mkdir models -for i in graphdef ; do +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_int32_int32_int32 models/. done @@ -667,7 +673,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_version_load_unload_disabled rm -fr models mkdir models -for i in graphdef ; do +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_int32_int32_int32 models/. done @@ -696,7 +702,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_model_modify rm -fr models config.pbtxt.* mkdir models -for i in savedmodel plan ; do +for i in libtorch plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. sed '/^version_policy/d' \ $DATADIR/qa_model_repository/${i}_float32_float32_float32/config.pbtxt > config.pbtxt.${i} @@ -731,7 +737,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_dynamic_file_delete rm -fr models config.pbtxt.* mkdir models -for i in savedmodel plan; do +for i in onnx plan; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done @@ -757,17 +763,18 @@ wait $SERVER_PID LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_multiple_model_repository_polling -rm -fr models models_0 savedmodel_float32_float32_float32 +rm -fr models models_0 libtorch_float32_float32_float32 mkdir models models_0 -for i in graphdef ; do +for i in openvino ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. + echo 'parameters { key: "ENABLE_BATCH_PADDING" value { string_value: "YES" } }' >> models/openvino_float32_float32_float32/config.pbtxt done for i in onnx ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 . -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 models/. && \ - rm -rf models/savedmodel_float32_float32_float32/3 +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 . +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 models/. && \ + rm -rf models/libtorch_float32_float32_float32/3 SERVER_ARGS="--model-repository=`pwd`/models --model-repository=`pwd`/models_0 \ --model-control-mode=poll --repository-poll-secs=1 --exit-timeout-secs=5" @@ -791,17 +798,18 @@ wait $SERVER_PID LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_multiple_model_repository_control -rm -fr models models_0 savedmodel_float32_float32_float32 +rm -fr models models_0 libtorch_float32_float32_float32 mkdir models models_0 -for i in graphdef ; do +for i in openvino ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. + echo 'parameters { key: "ENABLE_BATCH_PADDING" value { string_value: "YES" } }' >> models/openvino_float32_float32_float32/config.pbtxt done for i in onnx ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 . -cp -r $DATADIR/qa_model_repository/savedmodel_float32_float32_float32 models/. && \ - rm -rf models/savedmodel_float32_float32_float32/3 +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 . +cp -r $DATADIR/qa_model_repository/libtorch_float32_float32_float32 models/. && \ + rm -rf models/libtorch_float32_float32_float32/3 # Show model control mode will override deprecated model control options SERVER_ARGS="--model-repository=`pwd`/models --model-repository=`pwd`/models_0 \ @@ -933,8 +941,8 @@ for i in plan onnx ; do sed -i "s/max_batch_size:.*/max_batch_size: 1/" models_0/simple_${i}_float32_float32_float32/config.pbtxt done -# savedmodel doesn't load because it is duplicated in 2 repositories -for i in savedmodel ; do +# libtorch doesn't load because it is duplicated in 2 repositories +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done @@ -943,7 +951,7 @@ SERVER_ARGS="--model-repository=`pwd`/models --model-repository=`pwd`/models_0 \ --model-control-mode=explicit \ --strict-readiness=false \ --strict-model-config=false --exit-on-error=false \ - --load-model=savedmodel_float32_float32_float32 \ + --load-model=libtorch_float32_float32_float32 \ --load-model=plan_float32_float32_float32 \ --load-model=simple_onnx_float32_float32_float32" SERVER_LOG="./inference_server_$LOG_IDX.log" @@ -978,8 +986,8 @@ for i in plan onnx ; do sed -i "s/max_batch_size:.*/max_batch_size: 1/" models_0/simple_${i}_float32_float32_float32/config.pbtxt done -# savedmodel doesn't load because it is duplicated in 2 repositories -for i in savedmodel ; do +# libtorch doesn't load because it is duplicated in 2 repositories +for i in libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models_0/. done @@ -1070,7 +1078,7 @@ LOG_IDX=$((LOG_IDX+1)) rm -fr models models_0 config.pbtxt.* mkdir models models_0 # Ensemble models in the second repository -for i in graphdef savedmodel ; do +for i in plan libtorch ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. cp -r $DATADIR/qa_ensemble_model_repository/qa_model_repository/simple_${i}_float32_float32_float32 models_0/. done @@ -1086,8 +1094,8 @@ SERVER_ARGS="--model-repository=`pwd`/models --model-repository=`pwd`/models_0 \ --strict-readiness=false \ --strict-model-config=false --exit-on-error=false \ --load-model=onnx_float32_float32_float32 \ - --load-model=graphdef_float32_float32_float32 \ - --load-model=simple_savedmodel_float32_float32_float32" + --load-model=plan_float32_float32_float32 \ + --load-model=simple_libtorch_float32_float32_float32" SERVER_LOG="./inference_server_$LOG_IDX.log" run_server if [ "$SERVER_PID" == "0" ]; then @@ -1369,7 +1377,7 @@ done # Send HTTP request to control endpoint rm -fr models config.pbtxt.* mkdir models -for i in graphdef savedmodel onnx plan ; do +for i in openvino libtorch onnx plan ; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. done @@ -1388,7 +1396,7 @@ fi # unload API should return bad request set +e -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/repository/models/graphdef_float32_float32_float32/unload` +code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/repository/models/openvino_float32_float32_float32/unload` set -e if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" @@ -1397,7 +1405,7 @@ fi # the model should be available/ready set +e -code=`curl -s -w %{http_code} localhost:8000/v2/models/graphdef_float32_float32_float32/ready` +code=`curl -s -w %{http_code} localhost:8000/v2/models/openvino_float32_float32_float32/ready` set -e if [ "$code" != "200" ]; then echo -e "\n***\n*** Test Failed\n***" @@ -1405,11 +1413,11 @@ if [ "$code" != "200" ]; then fi # remove model file so that if reload is triggered, model will become unavailable -rm models/graphdef_float32_float32_float32/*/* +rm models/openvino_float32_float32_float32/*/* # load API should return bad request set +e -code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/repository/models/graphdef_float32_float32_float32/load` +code=`curl -s -w %{http_code} -o ./curl.out -X POST localhost:8000/v2/repository/models/openvino_float32_float32_float32/load` set -e if [ "$code" == "200" ]; then echo -e "\n***\n*** Test Failed\n***" @@ -1418,7 +1426,7 @@ fi # the model should be available/ready set +e -code=`curl -s -w %{http_code} localhost:8000/v2/models/graphdef_float32_float32_float32/ready` +code=`curl -s -w %{http_code} localhost:8000/v2/models/openvino_float32_float32_float32/ready` set -e if [ "$code" != "200" ]; then echo -e "\n***\n*** Test Failed\n***" @@ -1434,7 +1442,7 @@ LOG_IDX=$((LOG_IDX+1)) # some more comprehensive fuzz attacks. rm -fr models mkdir models -for i in graphdef ; do +for i in openvino ; do cp -r $DATADIR/qa_model_repository/${i}_int32_int32_int32 models/. done @@ -2136,7 +2144,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_add_custom_config rm -fr models config.pbtxt.* mkdir models -for i in savedmodel; do +for i in libtorch; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. mkdir models/${i}_float32_float32_float32/configs sed 's/^version_policy:.*/version_policy: { specific: { versions: [2] }}/' \ @@ -2168,7 +2176,7 @@ LOG_IDX=$((LOG_IDX+1)) # LifeCycleTest.test_delete_custom_config rm -fr models config.pbtxt.* mkdir models -for i in savedmodel; do +for i in libtorch; do cp -r $DATADIR/qa_model_repository/${i}_float32_float32_float32 models/. mkdir models/${i}_float32_float32_float32/configs sed 's/^version_policy:.*/version_policy: { specific: { versions: [2] }}/' \ diff --git a/qa/L0_long_running_stress/scenarios.py b/qa/L0_long_running_stress/scenarios.py index abb0004e90..650f228ab5 100755 --- a/qa/L0_long_running_stress/scenarios.py +++ b/qa/L0_long_running_stress/scenarios.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -84,10 +84,8 @@ def get_trial(self): def get_datatype(self, trial): # Get the datatype to use based on what models are available (see test.sh) - if ("plan" in trial) or ("savedmodel" in trial): + if "plan" in trial: return np.float32 - if "graphdef" in trial: - return np.dtype(object) return np.int32 # FIXME do we need client meta data? @@ -205,7 +203,7 @@ def __init__( # Add no validation models self.options_.append( PerfAnalyzerScenario.ModelOption( - "resnet_v1_50_graphdef_def", 32, (1, 4, 1), queue_latency_range_us + "resnet_v1_50_def", 32, (1, 4, 1), queue_latency_range_us ) ) for trial in sequence_trials: @@ -280,34 +278,19 @@ def generate_sequence_data(self, trial, dtype, data_filename): input_data.append({input0: [res]}) output0 = "OUTPUT" if "libtorch" not in trial else "OUTPUT__0" output_data = [] - if ("savedmodel" in trial) and ("nobatch" in trial): - # Special case where the model is accumulator - sum = 0 - for i in range(3): - sum += i - if dtype == np.float32: - res = float(sum) - elif dtype == np.int32: - res = sum - elif dtype == np.dtype(object): - res = str(sum) - else: - raise Exception("unexpected sequence data type {}".format(dtype)) - output_data.append({output0: [res]}) - else: - for i in range(3): - res = 1 if i == 0 else i - if dtype == np.float32: - res = float(res) - elif dtype == np.int32: - res = int(res) - elif dtype == np.dtype(object): - res = str(res) - else: - raise Exception("unexpected sequence data type {}".format(dtype)) - output_data.append( - {output0: [res if dtype != np.dtype(object) else str(res)]} - ) + for i in range(3): + res = 1 if i == 0 else i + if dtype == np.float32: + res = float(res) + elif dtype == np.int32: + res = int(res) + elif dtype == np.dtype(object): + res = str(res) + else: + raise Exception("unexpected sequence data type {}".format(dtype)) + output_data.append( + {output0: [res if dtype != np.dtype(object) else str(res)]} + ) data = {"data": [input_data]} data["validation_data"] = [output_data] @@ -351,7 +334,7 @@ def run(self, client_metadata): class ResNetScenario(Scenario): def __init__(self, name, batch_size=32, verbose=False, out_stream=sys.stdout): super().__init__(name, [], verbose, out_stream) - self.model_name_ = "resnet_v1_50_graphdef_def" + self.model_name_ = "resnet_v1_50_def" self.batch_size_ = batch_size img = self.preprocess("../images/vulture.jpeg") @@ -525,7 +508,6 @@ def get_expected_result(self, expected_result, value, trial, flag_str=None): # information. if ( ("nobatch" not in trial and ("custom" not in trial)) - or ("graphdef" in trial) or ("plan" in trial) or ("onnx" in trial) ) or ("libtorch" in trial): @@ -555,9 +537,7 @@ def check_sequence_async( """ if ( - ("savedmodel" not in trial) - and ("graphdef" not in trial) - and ("custom" not in trial) + ("custom" not in trial) and ("onnx" not in trial) and ("libtorch" not in trial) and ("plan" not in trial) diff --git a/qa/L0_long_running_stress/stress.py b/qa/L0_long_running_stress/stress.py index 978f204ee6..ae67d27668 100755 --- a/qa/L0_long_running_stress/stress.py +++ b/qa/L0_long_running_stress/stress.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,7 @@ FLAGS = None CORRELATION_ID_BLOCK_SIZE = 1024 * 1024 -BACKENDS = os.environ.get("BACKENDS", "graphdef savedmodel onnx plan") +BACKENDS = os.environ.get("BACKENDS", "onnx plan") _thread_exceptions = [] _thread_exceptions_mutex = threading.Lock() @@ -66,7 +66,7 @@ def get_trials(is_sequence=True): _trials = () if is_sequence: for backend in BACKENDS.split(" "): - if (backend != "libtorch") and (backend != "savedmodel"): + if backend != "libtorch": _trials += (backend + "_nobatch",) _trials += (backend,) else: diff --git a/qa/L0_long_running_stress/test.sh b/qa/L0_long_running_stress/test.sh index b98a89f955..1973028bcf 100755 --- a/qa/L0_long_running_stress/test.sh +++ b/qa/L0_long_running_stress/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -63,7 +63,7 @@ fi RET=0 # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch"} +BACKENDS=${BACKENDS:="onnx libtorch"} export BACKENDS export CI_JOB_ID=${CI_JOB_ID} @@ -75,10 +75,8 @@ rm -fr *.log *.txt models validation_data csv_dir && mkdir models validation_da # Get the datatype to use based on the backend function get_datatype () { local dtype='int32' - if [[ $1 == "plan" ]] || [[ $1 == "savedmodel" ]]; then + if [[ $1 == "plan" ]]; then dtype='float32' - elif [[ $1 == "graphdef" ]]; then - dtype='object' fi echo $dtype } @@ -132,9 +130,9 @@ cp -r ../custom_models/custom_zero_1_float32 $MODEL_DIR/custom_zero_1_float32 && echo "{ key: \"execute_delay_ms\"; value: { string_value: \"10000\" }}" >> config.pbtxt && \ echo "]" >> config.pbtxt) -cp -r $DATADIR/tf_model_store/resnet_v1_50_graphdef $MODEL_DIR/resnet_v1_50_graphdef_def && \ - (cd $MODEL_DIR/resnet_v1_50_graphdef_def && \ - sed -i 's/^name: "resnet_v1_50_graphdef"/name: "resnet_v1_50_graphdef_def"/' config.pbtxt && \ +cp -r $DATADIR/onnx_model_store/resnet_v1_50 $MODEL_DIR/resnet_v1_50_def && \ + (cd $MODEL_DIR/resnet_v1_50_def && \ + sed -i 's/^name: "resnet_v1_50"/name: "resnet_v1_50_def"/' config.pbtxt && \ echo "optimization { }" >> config.pbtxt) SERVER_ARGS="--model-repository=`pwd`/$MODEL_DIR" diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/config.pbtxt deleted file mode 100644 index b393fb4e00..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16, 1 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/expected deleted file mode 100644 index 9db37f7864..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_dims/expected +++ /dev/null @@ -1 +0,0 @@ -Internal: unable to autofill for 'bad_input_dims', model tensor configurations are contradicting each other in terms of whether batching is supported \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/config.pbtxt deleted file mode 100644 index e11f302484..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_FP32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/expected deleted file mode 100644 index 584634b2eb..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_input_type/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: unable to load model 'bad_input_type', configuration expects datatype TYPE_FP32 for input 'INPUT1', model provides TYPE_INT32 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/config.pbtxt deleted file mode 100644 index 004ed9a54f..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 1 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/expected deleted file mode 100644 index 70a0138e77..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_dims/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: model 'bad_output_dims', tensor 'OUTPUT1': the model expects 2 dimensions (shape \[-1,16\]) but the model configuration specifies 2 dimensions (an initial batch dimension because max_batch_size > 0 followed by the explicit tensor shape, making complete shape \[-1,1\]) \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/config.pbtxt deleted file mode 100644 index 45e4ef89f4..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT16 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/expected deleted file mode 100644 index bbbe1846d1..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/bad_output_type/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: unable to load model 'bad_output_type', configuration expects datatype TYPE_INT16 for output 'OUTPUT0', model provides TYPE_INT8 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/config.pbtxt deleted file mode 100644 index cee3e28b89..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/config.pbtxt +++ /dev/null @@ -1,30 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/expected deleted file mode 100644 index caaebb93a0..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/too_many_inputs/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: unable to load model 'too_many_inputs', configuration expects 3 inputs, model provides 2 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/config.pbtxt deleted file mode 100644 index 0df318caa8..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT_UNKNOWN" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/expected deleted file mode 100644 index 3f101c14fa..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_input/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: unexpected inference input 'INPUT_UNKNOWN', allowed inputs are: INPUT0, INPUT1 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/config.pbtxt b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/config.pbtxt deleted file mode 100644 index 979b05c4ee..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/config.pbtxt +++ /dev/null @@ -1,20 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT_UNKNOWN" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/expected b/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/expected deleted file mode 100644 index a525ae910b..0000000000 --- a/qa/L0_model_config/autofill_noplatform/tensorflow_savedmodel/unknown_output/expected +++ /dev/null @@ -1 +0,0 @@ -Invalid argument: unexpected inference output 'OUTPUT_UNKNOWN', allowed outputs are: OUTPUT0, OUTPUT1 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/1/model.graphdef b/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/1/model.graphdef deleted file mode 100644 index 67c09170e5..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/1/model.graphdef and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/config.pbtxt deleted file mode 100644 index 49b49aadbb..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/expected deleted file mode 100644 index ed01acd5e0..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/no_name_platform/expected +++ /dev/null @@ -1,45 +0,0 @@ -name: "no_name_platform" -platform: "tensorflow_graphdef" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_name_platform" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.graphdef" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/config.pbtxt deleted file mode 100644 index b3bc21377e..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/config.pbtxt +++ /dev/null @@ -1,41 +0,0 @@ -name: "reshape_config_provided" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/expected deleted file mode 100644 index 51e2d46d42..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/expected +++ /dev/null @@ -1,59 +0,0 @@ -name: "reshape_config_provided" -platform: "tensorflow_graphdef" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -instance_group { - name: "reshape_config_provided" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.graphdef" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/config.pbtxt deleted file mode 100644 index bf4222124a..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/config.pbtxt +++ /dev/null @@ -1,44 +0,0 @@ -name: "cpu_instance" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "cpu_instance" - kind: KIND_CPU -} -dynamic_batching { -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected deleted file mode 100644 index f60d0950f1..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected +++ /dev/null @@ -1,47 +0,0 @@ -name: "cpu_instance" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "cpu_instance" - count: 2 - kind: KIND_CPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.1 deleted file mode 100644 index dfcf1c7e89..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.1 +++ /dev/null @@ -1,47 +0,0 @@ -name: "cpu_instance" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "cpu_instance" - count: 2 - kind: KIND_CPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.2 deleted file mode 100644 index 03a9721822..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.2 +++ /dev/null @@ -1,47 +0,0 @@ -name: "cpu_instance" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "cpu_instance" - count: 2 - kind: KIND_CPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.3 deleted file mode 100644 index 4d69237a2e..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/cpu_instance/expected.3 +++ /dev/null @@ -1,47 +0,0 @@ -name: "cpu_instance" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "cpu_instance" - count: 2 - kind: KIND_CPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/config.pbtxt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected deleted file mode 100644 index abbc108196..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "empty_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "empty_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.1 deleted file mode 100644 index 164b3afd2f..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "empty_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "empty_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.2 deleted file mode 100644 index 6ad6e0d311..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "empty_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "empty_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.3 deleted file mode 100644 index 9298a2dc33..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/empty_config/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "empty_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "empty_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/config.pbtxt deleted file mode 100644 index e2c3c36d49..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/config.pbtxt +++ /dev/null @@ -1,10 +0,0 @@ -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - }, - { - name: "OUTPUT1" - dims: [ -1, 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected deleted file mode 100644 index 5ba092a8ad..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_1" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_1" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.1 deleted file mode 100644 index a2db3d6b62..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_1" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_1" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.2 deleted file mode 100644 index f847b58097..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_1" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_1" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.3 deleted file mode 100644 index 0e09e46e87..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_1/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_1" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_1" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/config.pbtxt deleted file mode 100644 index a4faf54369..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/config.pbtxt +++ /dev/null @@ -1,10 +0,0 @@ -output [ - { - name: "OUTPUT1" - dims: [ -1, 16 ] - }, - { - name: "OUTPUT0" - data_type: TYPE_INT8 - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected deleted file mode 100644 index 137a62f2c1..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_2" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_2" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.1 deleted file mode 100644 index fd9da45fc4..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_2" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_2" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.2 deleted file mode 100644 index efbb5a2a0c..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_2" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_2" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.3 deleted file mode 100644 index 27fa02d910..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/hint_for_no_batch_2/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "hint_for_no_batch_2" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: -1 - dims: 16 -} -instance_group { - name: "hint_for_no_batch_2" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/config.pbtxt deleted file mode 100644 index 29ee883a4b..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/config.pbtxt +++ /dev/null @@ -1,10 +0,0 @@ -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - }, - { - name: "INPUT1" - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected deleted file mode 100644 index 42eb4b0821..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_input" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_input" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.1 deleted file mode 100644 index c5925abf6b..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_input" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_input" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.2 deleted file mode 100644 index 0951a6ceaf..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_input" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_input" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.3 deleted file mode 100644 index c2e88938bb..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_input/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_input" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_input" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/config.pbtxt deleted file mode 100644 index fa9cc35967..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/config.pbtxt +++ /dev/null @@ -1,10 +0,0 @@ -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - }, - { - name: "OUTPUT1" - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected deleted file mode 100644 index 2e1f32882f..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_output" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_output" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.1 deleted file mode 100644 index cf9d68e891..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_output" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_output" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.2 deleted file mode 100644 index 48deb2c7fe..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_output" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_output" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.3 deleted file mode 100644 index c3f49cdfd7..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/incomplete_output/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "incomplete_output" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "incomplete_output" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/config.pbtxt deleted file mode 100644 index 78cc4480b8..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/config.pbtxt +++ /dev/null @@ -1,3 +0,0 @@ -instance_group { - kind: KIND_MODEL -} diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected deleted file mode 100644 index 7f1b142e3b..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected +++ /dev/null @@ -1,47 +0,0 @@ -name: "kind_model_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "kind_model_config_0" - count: 1 - kind: KIND_MODEL -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.1 deleted file mode 100644 index 61cfcc6a23..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.1 +++ /dev/null @@ -1,47 +0,0 @@ -name: "kind_model_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "kind_model_config_0" - count: 1 - kind: KIND_MODEL -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.2 deleted file mode 100644 index 4b0ddbeb8e..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.2 +++ /dev/null @@ -1,47 +0,0 @@ -name: "kind_model_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "kind_model_config_0" - count: 1 - kind: KIND_MODEL -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.3 deleted file mode 100644 index abea687937..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/kind_model_config/expected.3 +++ /dev/null @@ -1,47 +0,0 @@ -name: "kind_model_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "kind_model_config_0" - count: 1 - kind: KIND_MODEL -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/config.pbtxt deleted file mode 100644 index 1cf214cafe..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/config.pbtxt +++ /dev/null @@ -1 +0,0 @@ -max_batch_size: 8 \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected deleted file mode 100644 index fcf0de4262..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "max_batch_size_set" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "max_batch_size_set" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.1 deleted file mode 100644 index 4b1dc1abd2..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "max_batch_size_set" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "max_batch_size_set" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.2 deleted file mode 100644 index 9acbbe3f12..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "max_batch_size_set" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "max_batch_size_set" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.3 deleted file mode 100644 index e129508a01..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/max_batch_size_set/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "max_batch_size_set" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "max_batch_size_set" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected deleted file mode 100644 index 2250f91f71..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected +++ /dev/null @@ -1,48 +0,0 @@ -name: "no_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.1 deleted file mode 100644 index 56c1221734..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.1 +++ /dev/null @@ -1,48 +0,0 @@ -name: "no_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.2 deleted file mode 100644 index 30875b1998..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.2 +++ /dev/null @@ -1,48 +0,0 @@ -name: "no_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.3 deleted file mode 100644 index 469b9aff76..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config/expected.3 +++ /dev/null @@ -1,48 +0,0 @@ -name: "no_config" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 4 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_config" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 4 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/1/model.savedmodel/saved_model.pb deleted file mode 100644 index e27067d53c..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/config.pbtxt deleted file mode 100644 index 5913902a76..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/config.pbtxt +++ /dev/null @@ -1,5 +0,0 @@ -instance_group [ - { - kind: KIND_CPU - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/expected deleted file mode 100644 index 165300aa9b..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_config_no_batch/expected +++ /dev/null @@ -1,41 +0,0 @@ -name: "no_config_no_batch" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -input { - name: "input" - data_type: TYPE_FP32 - dims: 1 - dims: 256 - dims: 256 - dims: 256 - dims: 1 -} -output { - name: "output" - data_type: TYPE_FP32 - dims: 1 - dims: 256 - dims: 256 - dims: 256 - dims: 14 -} -instance_group { - name: "no_config_no_batch_0" - count: 2 - kind: KIND_CPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/config.pbtxt deleted file mode 100644 index 49b49aadbb..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/config.pbtxt +++ /dev/null @@ -1,25 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT0" - data_type: TYPE_INT32 - dims: [ 16 ] - }, - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected deleted file mode 100644 index 393000147a..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected +++ /dev/null @@ -1,45 +0,0 @@ -name: "no_name_platform" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_name_platform" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.1 deleted file mode 100644 index 1a9c47cca7..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.1 +++ /dev/null @@ -1,45 +0,0 @@ -name: "no_name_platform" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_name_platform" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.2 deleted file mode 100644 index c47e51aeb3..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.2 +++ /dev/null @@ -1,45 +0,0 @@ -name: "no_name_platform" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_name_platform" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.3 deleted file mode 100644 index 42adbbf4d3..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/no_name_platform/expected.3 +++ /dev/null @@ -1,45 +0,0 @@ -name: "no_name_platform" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "no_name_platform" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/config.pbtxt deleted file mode 100644 index 95f67e119e..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/config.pbtxt +++ /dev/null @@ -1,34 +0,0 @@ -name: "reshape_config_provided" -max_batch_size: 8 -input [ - { - name: "INPUT0" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: [ 1 ] - reshape: { shape: [ ] } - } -] -input [ - { - name: "INPUT1" - data_type: TYPE_FP32 - dims: [ 8 ] - reshape: { shape: [ 4,1,2 ] } - } -] -output [ - { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: [ 8 ] - reshape: { shape: [ 4,1,2 ] } - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected deleted file mode 100644 index 4fd8a8edb9..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected +++ /dev/null @@ -1,62 +0,0 @@ -name: "reshape_config_provided" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -instance_group { - name: "reshape_config_provided" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.1 deleted file mode 100644 index 87d646c314..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.1 +++ /dev/null @@ -1,62 +0,0 @@ -name: "reshape_config_provided" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -instance_group { - name: "reshape_config_provided" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.2 deleted file mode 100644 index 3605cb1fc0..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.2 +++ /dev/null @@ -1,62 +0,0 @@ -name: "reshape_config_provided" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -instance_group { - name: "reshape_config_provided" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.3 deleted file mode 100644 index c273096707..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/expected.3 +++ /dev/null @@ -1,62 +0,0 @@ -name: "reshape_config_provided" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 8 -input { - name: "INPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -input { - name: "INPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -output { - name: "OUTPUT0" - data_type: TYPE_FP32 - dims: 1 - reshape { - } -} -output { - name: "OUTPUT1" - data_type: TYPE_FP32 - dims: 8 - reshape { - shape: 4 - shape: 1 - shape: 2 - } -} -instance_group { - name: "reshape_config_provided" - count: 1 - gpus: 0 - kind: KIND_GPU -} -dynamic_batching { - preferred_batch_size: 8 -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" \ No newline at end of file diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/1/model.savedmodel/saved_model.pb b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/1/model.savedmodel/saved_model.pb deleted file mode 100644 index a76abafbf7..0000000000 Binary files a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/config.pbtxt b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/config.pbtxt deleted file mode 100644 index 2814fb7e5c..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/config.pbtxt +++ /dev/null @@ -1,20 +0,0 @@ -max_batch_size: 1 -input [ - { - name: "INPUT1" - data_type: TYPE_INT32 - dims: [ 16 ] - } -] -output [ - { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: [ 16 ] - }, - { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: [ 16 ] - } -] diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected deleted file mode 100644 index c41ed15143..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected +++ /dev/null @@ -1,45 +0,0 @@ -name: "too_few_inputs" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "too_few_inputs" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.1 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.1 deleted file mode 100644 index 0a4b67356d..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.1 +++ /dev/null @@ -1,45 +0,0 @@ -name: "too_few_inputs" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "too_few_inputs" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.2 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.2 deleted file mode 100644 index 626db7022b..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.2 +++ /dev/null @@ -1,45 +0,0 @@ -name: "too_few_inputs" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "too_few_inputs" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.3 b/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.3 deleted file mode 100644 index 5c93813b17..0000000000 --- a/qa/L0_model_config/autofill_noplatform_success/tensorflow_savedmodel/too_few_inputs/expected.3 +++ /dev/null @@ -1,45 +0,0 @@ -name: "too_few_inputs" -platform: "tensorflow_savedmodel" -version_policy { - latest { - num_versions: 1 - } -} -max_batch_size: 1 -input { - name: "INPUT0" - data_type: TYPE_INT32 - dims: 16 -} -input { - name: "INPUT1" - data_type: TYPE_INT32 - dims: 16 -} -output { - name: "OUTPUT0" - data_type: TYPE_INT8 - dims: 16 -} -output { - name: "OUTPUT1" - data_type: TYPE_INT8 - dims: 16 -} -instance_group { - name: "too_few_inputs" - count: 1 - gpus: 0 - kind: KIND_GPU -} -default_model_filename: "model.savedmodel" -optimization { - input_pinned_memory { - enable: true - } - output_pinned_memory { - enable: true - } -} -backend: "tensorflow" -runtime: "" diff --git a/qa/L0_model_config/noautofill_platform/control_kind_end_multiple/config.pbtxt b/qa/L0_model_config/noautofill_platform/control_kind_end_multiple/config.pbtxt index 1fecf68202..ddbf92d2dd 100644 --- a/qa/L0_model_config/noautofill_platform/control_kind_end_multiple/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/control_kind_end_multiple/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "control_kind_end_multiple" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 sequence_batching { control_input [ diff --git a/qa/L0_model_config/noautofill_platform/control_kind_ready_multiple/config.pbtxt b/qa/L0_model_config/noautofill_platform/control_kind_ready_multiple/config.pbtxt index 82f35e2aa0..e61c7441c5 100644 --- a/qa/L0_model_config/noautofill_platform/control_kind_ready_multiple/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/control_kind_ready_multiple/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "control_kind_ready_multiple" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 sequence_batching { control_input [ diff --git a/qa/L0_model_config/noautofill_platform/control_kind_start_multiple/config.pbtxt b/qa/L0_model_config/noautofill_platform/control_kind_start_multiple/config.pbtxt index 83ae70256e..2145f193ce 100644 --- a/qa/L0_model_config/noautofill_platform/control_kind_start_multiple/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/control_kind_start_multiple/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "control_kind_start_multiple" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 sequence_batching { control_input [ diff --git a/qa/L0_model_config/noautofill_platform/control_tensor_multiple/config.pbtxt b/qa/L0_model_config/noautofill_platform/control_tensor_multiple/config.pbtxt index a9bcf6680f..3dab9279a4 100644 --- a/qa/L0_model_config/noautofill_platform/control_tensor_multiple/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/control_tensor_multiple/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "control_tensor_multiple" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 sequence_batching { control_input [ diff --git a/qa/L0_model_config/noautofill_platform/control_tensor_no_value/config.pbtxt b/qa/L0_model_config/noautofill_platform/control_tensor_no_value/config.pbtxt index a4763b6f59..f7d6c70c1b 100644 --- a/qa/L0_model_config/noautofill_platform/control_tensor_no_value/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/control_tensor_no_value/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "control_tensor_no_value" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 sequence_batching { control_input [ diff --git a/qa/L0_model_config/noautofill_platform/default_priority_level0/config.pbtxt b/qa/L0_model_config/noautofill_platform/default_priority_level0/config.pbtxt index 7c29e74637..174177e8c3 100644 --- a/qa/L0_model_config/noautofill_platform/default_priority_level0/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/default_priority_level0/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "default_priority_level0" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { priority_levels: 3 diff --git a/qa/L0_model_config/noautofill_platform/default_priority_level1/config.pbtxt b/qa/L0_model_config/noautofill_platform/default_priority_level1/config.pbtxt index 87cffd5c84..3c442ca3a8 100644 --- a/qa/L0_model_config/noautofill_platform/default_priority_level1/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/default_priority_level1/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "default_priority_level1" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { priority_levels: 3 diff --git a/qa/L0_model_config/noautofill_platform/preserve_ordering0/config.pbtxt b/qa/L0_model_config/noautofill_platform/preserve_ordering0/config.pbtxt index 3d7c0a6fd9..5ce7428215 100644 --- a/qa/L0_model_config/noautofill_platform/preserve_ordering0/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/preserve_ordering0/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "preserve_ordering0" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { preserve_ordering: true diff --git a/qa/L0_model_config/noautofill_platform/preserve_ordering1/config.pbtxt b/qa/L0_model_config/noautofill_platform/preserve_ordering1/config.pbtxt index 82a8948817..cc248bb114 100644 --- a/qa/L0_model_config/noautofill_platform/preserve_ordering1/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/preserve_ordering1/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "preserve_ordering1" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { preserve_ordering: true diff --git a/qa/L0_model_config/noautofill_platform/preserve_ordering2/config.pbtxt b/qa/L0_model_config/noautofill_platform/preserve_ordering2/config.pbtxt index 35ff408967..1a4482de51 100644 --- a/qa/L0_model_config/noautofill_platform/preserve_ordering2/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/preserve_ordering2/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "preserve_ordering2" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { preserve_ordering: true diff --git a/qa/L0_model_config/noautofill_platform/priority_level0/config.pbtxt b/qa/L0_model_config/noautofill_platform/priority_level0/config.pbtxt index 7167b043d2..d57fe998d8 100644 --- a/qa/L0_model_config/noautofill_platform/priority_level0/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/priority_level0/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "priority_level0" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { priority_levels: 3 diff --git a/qa/L0_model_config/noautofill_platform/priority_level1/config.pbtxt b/qa/L0_model_config/noautofill_platform/priority_level1/config.pbtxt index aa274bd9b3..cce2836416 100644 --- a/qa/L0_model_config/noautofill_platform/priority_level1/config.pbtxt +++ b/qa/L0_model_config/noautofill_platform/priority_level1/config.pbtxt @@ -1,5 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "priority_level1" -platform: "tensorflow_savedmodel" +platform: "onnxruntime" max_batch_size: 8 dynamic_batching { priority_levels: 3 diff --git a/qa/L0_model_config/special_cases/invalid_platform/config.pbtxt b/qa/L0_model_config/special_cases/invalid_platform/config.pbtxt index 6cdb34f1c0..8d5af886a9 100644 --- a/qa/L0_model_config/special_cases/invalid_platform/config.pbtxt +++ b/qa/L0_model_config/special_cases/invalid_platform/config.pbtxt @@ -1,6 +1,31 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "invalid_platform" -platform: "tensorflo" -default_model_filename: "model.savedmodel" +platform: "onnxruntim" +default_model_filename: "model.onnx" max_batch_size: 8 input [ { diff --git a/qa/L0_model_config/special_cases/invalid_platform/expected b/qa/L0_model_config/special_cases/invalid_platform/expected index 4be7d00ff0..697a4ee704 100644 --- a/qa/L0_model_config/special_cases/invalid_platform/expected +++ b/qa/L0_model_config/special_cases/invalid_platform/expected @@ -1 +1 @@ -unexpected platform type 'tensorflo' for invalid_platform +unexpected 'platform' and 'backend' pair, got:onnxruntim, onnxruntime \ No newline at end of file diff --git a/qa/L0_model_config/special_cases/noautofill_noconfig/expected b/qa/L0_model_config/special_cases/noautofill_noconfig/expected deleted file mode 100644 index 5a0abf84dc..0000000000 --- a/qa/L0_model_config/special_cases/noautofill_noconfig/expected +++ /dev/null @@ -1 +0,0 @@ -model configuration is not provided diff --git a/qa/L0_model_config/test.sh b/qa/L0_model_config/test.sh index 6071547de1..7622cf93b2 100755 --- a/qa/L0_model_config/test.sh +++ b/qa/L0_model_config/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,7 @@ source ../common/util.sh export CUDA_VISIBLE_DEVICES=0 -TRIALS="tensorflow_savedmodel tensorflow_graphdef tensorrt_plan onnxruntime_onnx pytorch_libtorch" +TRIALS="tensorrt_plan onnxruntime_onnx pytorch_libtorch" # Copy fixed TensorRT plans into the test model repositories. for modelpath in \ @@ -259,13 +259,9 @@ done # Copy other required models mkdir -p special_cases/invalid_platform/1 -cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/savedmodel_float32_float32_float32/1/model.savedmodel \ - special_cases/invalid_platform/1/ -# Note that graphdef models don't support auto-complete-config -# and that is why we are using graphdef model in this test case. -mkdir -p special_cases/noautofill_noconfig/1 -cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/graphdef_float32_float32_float32/1/model.graphdef \ - special_cases/noautofill_noconfig/1/ +cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/onnx_float32_float32_float32/1/model.onnx \ + special_cases/invalid_platform/1/ + # Create runtime escape scenario mkdir -p special_cases/runtime_escape/1 special_cases/runtime_escape/dummy_runtime touch special_cases/runtime_escape/dummy_runtime/libtriton_identity.so @@ -273,14 +269,6 @@ touch special_cases/runtime_escape/dummy_runtime/libtriton_identity.so mkdir -p special_cases/invalid_runtime/1 # Copy reshape model files into the test model repositories. -mkdir -p autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/1 -cp /data/inferenceserver/${REPO_VERSION}/qa_reshape_model_repository/graphdef_zero_2_float32/1/model.graphdef \ - autofill_noplatform_success/tensorflow_graphdef/reshape_config_provided/1 - -mkdir -p autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/1 -cp -r /data/inferenceserver/${REPO_VERSION}/qa_reshape_model_repository/savedmodel_zero_2_float32/1/model.savedmodel \ - autofill_noplatform_success/tensorflow_savedmodel/reshape_config_provided/1 - mkdir -p autofill_noplatform_success/tensorrt/reshape_config_provided/1 cp /data/inferenceserver/${REPO_VERSION}/qa_reshape_model_repository/plan_zero_4_float32/1/model.plan \ autofill_noplatform_success/tensorrt/reshape_config_provided/1 @@ -415,34 +403,6 @@ for TARGET in `ls special_cases`; do fi done -# Run noautofill unittest -SERVER_ARGS="--model-repository=`pwd`/models --model-control-mode=explicit --log-verbose=1" -SERVER_LOG=$SERVER_LOG_BASE.special_case_noautofill_test.log - -rm -fr models && mkdir models -cp -r special_cases/noautofill_noconfig models/. - -echo -e "Test on special_cases/noautofill_test" >> $CLIENT_LOG - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python noautofill_test.py >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Python NoAutoFill Test Failed\n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - for TRIAL in $TRIALS; do # Run all tests that require no autofill but that add the platform to # the model config before running the test diff --git a/qa/L0_multi_server/test.sh b/qa/L0_multi_server/test.sh index cd5ff3d407..d81041e8cb 100755 --- a/qa/L0_multi_server/test.sh +++ b/qa/L0_multi_server/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ RET=0 MULTI_SERVER=multi_server CLIENT_LOG=$MULTI_SERVER MULTI_SERVER=./$MULTI_SERVER -BACKENDS=(graphdef onnx plan) +BACKENDS=(onnx plan) THREAD_COUNT=32 LOOPS=32 diff --git a/qa/L0_nullchar_string/nullchar_string_client.py b/qa/L0_nullchar_string/nullchar_string_client.py index 1ab76bcf03..812e17ce95 100755 --- a/qa/L0_nullchar_string/nullchar_string_client.py +++ b/qa/L0_nullchar_string/nullchar_string_client.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -89,9 +89,17 @@ # Send inference request to the inference server. Get results for # output tensor. + input_name = "INPUT0" + output_name = "OUTPUT0" + + # If using libtorch model, set input and output name to "INPUT__0" and "OUTPUT__0" + if "libtorch" in FLAGS.model_name: + input_name = "INPUT__0" + output_name = "OUTPUT__0" + inputs = [ client_util.InferInput( - "INPUT0", input0_data.shape, np_to_triton_dtype(np.object_) + input_name, input0_data.shape, np_to_triton_dtype(np.object_) ) ] inputs[0].set_data_from_numpy(input0_data) @@ -100,7 +108,7 @@ # We expect there to be 1 result (with batch-size 1). Compare the input # and output tensor calculated by the model. They must be the same. - output0_data = results.as_numpy("OUTPUT0") + output0_data = results.as_numpy(output_name) print(input0_data, "?=?", output0_data) assert np.equal(input0_data.astype(np.bytes_), output0_data).all() diff --git a/qa/L0_nullchar_string/test.sh b/qa/L0_nullchar_string/test.sh index bded41dc92..5dced6c5d0 100755 --- a/qa/L0_nullchar_string/test.sh +++ b/qa/L0_nullchar_string/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -42,7 +42,7 @@ export CUDA_VISIBLE_DEVICES=0 CLIENT_LOG="./client.log" DATADIR=/data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository -MODELS="graphdef_nobatch_zero_1_object savedmodel_nobatch_zero_1_object" +MODELS="python_string libtorch_nobatch_zero_1_object" NULLCHAR_CLIENT_PY=nullchar_string_client.py SERVER=/opt/tritonserver/bin/tritonserver @@ -53,9 +53,15 @@ source ../common/util.sh rm -f $CLIENT_LOG $SERVER_LOG models mkdir -p models -for MODEL in $MODELS; do - cp -r $DATADIR/$MODEL models/. -done + +# Copy the python model +mkdir -p models/python_string/1/ +cp -fr ../python_models/string/model.py models/python_string/1/ +cp ../python_models/string/config.pbtxt models/python_string +sed -i 's/name: "string"/name: "python_string"/' models/python_string/config.pbtxt + +# Copy the libtorch model +cp -r $DATADIR/libtorch_nobatch_zero_1_object models/. run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/L0_orca/orca_http_test.py b/qa/L0_orca/orca_http_test.py new file mode 100755 index 0000000000..936fb97ae1 --- /dev/null +++ b/qa/L0_orca/orca_http_test.py @@ -0,0 +1,161 @@ +#!/usr/bin/python3 +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import json +import sys + +import requests + + +# To run the test, have tritonserver running and run this script with the endpoint as a flag. +# +# Example: +# ``` +# python3 orca_header_test.py http://localhost:8000/v2/models/ensemble/generate +# ``` +def get_endpoint_header(url, data, request_header=None): + """ + Sends a POST request to the given URL with the provided data and returns the value of the "endpoint-load-metrics" header, + or None if the request fails. + """ + HEADER_KEY = "endpoint-load-metrics" + try: + response = None + if request_header: + response = requests.post(url, json=data, headers=request_header) + else: + response = requests.post(url, json=data) + response.raise_for_status() + return response.headers.get(HEADER_KEY, "") + except requests.exceptions.RequestException as e: + print(f"Error making request: {e}") + return None + + +def parse_header_data(header, orca_format): + """ + Parses the header data into a dictionary based on the given format. + """ + METRIC_KEY = "named_metrics" + try: + if orca_format == "json": + # Parse the header in JSON format + data = json.loads(header.replace("JSON ", "")) + if METRIC_KEY in data: + return data[METRIC_KEY] + else: + print(f"No key '{METRIC_KEY}' in header data: {data}") + return None + elif orca_format == "text": + # Parse the header in TEXT format + data = {} + for key_value_pair in header.replace("TEXT ", "").split(", "): + key, value = key_value_pair.split("=") + if "." in key: + prefix, nested_key = key.split(".", 1) + if prefix == METRIC_KEY: + data[nested_key] = float(value) + if not data: + print(f"Could not parse any keys from header: {header}") + return None + return data + else: + print(f"Invalid ORCA format: {orca_format}") + return None + except (json.JSONDecodeError, ValueError, KeyError): + print("Error: Invalid data in the header.") + return None + + +def check_for_keys(data, desired_keys, orca_format): + """ + Checks if all desired keys are present in the given data dictionary. + """ + if all(key in data for key in desired_keys): + print( + f"ORCA header present in {orca_format} format with kv_cache_utilization: {[f'{k}: {data[k]}' for k in desired_keys]}" + ) + return True + else: + print(f"Missing keys in header: {', '.join(set(desired_keys) - set(data))}") + return False + + +def request_header(orca_format): + return {"endpoint-load-metrics-format": orca_format} if orca_format else None + + +def test_header_type(url, data, orca_format): + req_header = request_header(orca_format) + response_header = get_endpoint_header(args.url, TEST_DATA, req_header) + + desired_keys = { + "kv_cache_utilization", + "max_token_capacity", + } # Just the keys, no need to initialize with None + + if response_header is None: + print(f"Request to endpoint: '{args.url}' failed.") + return False + elif response_header == "": + if orca_format: + print( + f"response header empty, endpoint-load-metrics-format={orca_format} is not a valid ORCA metric format" + ) + return False + else: + # No request header set <=> no response header. Intended behavior. + print(f"response header empty, endpoint-load-metrics-format is not set") + return True + + data = parse_header_data(response_header, orca_format) + if data: + return check_for_keys(data, desired_keys, orca_format) + else: + print(f"Unexpected response header value: {response_header}") + return False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Make a POST request to generate endpoint to test the ORCA metrics header." + ) + parser.add_argument("url", help="The model URL to send the request to.") + args = parser.parse_args() + TEST_DATA = json.loads( + '{"text_input": "hello world", "max_tokens": 20, "bad_words": "", "stop_words": ""}' + ) + passed = True + + for format in ["json", "text", None]: + print("Checking response header for ORCA format:", format) + if not test_header_type(args.url, TEST_DATA, format): + print("FAIL on format:", format) + passed = False + + sys.exit(0 if passed else 1) diff --git a/qa/L0_orca/test.sh b/qa/L0_orca/test.sh new file mode 100755 index 0000000000..6069a75048 --- /dev/null +++ b/qa/L0_orca/test.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +RET=0 +BASE_DIR=$(pwd) +NUM_GPUS=${NUM_GPUS:=1} +TENSORRTLLM_BACKEND_REPO_TAG=${TENSORRTLLM_BACKEND_REPO_TAG:="main"} +TRITON_REPO_ORG=${TRITON_REPO_ORG:="https://github.com/triton-inference-server"} +TRT_ROOT="/usr/local/tensorrt" + +MODEL_NAME="gpt2_tensorrt_llm" +NAME="tensorrt_llm_benchmarking_test" +MODEL_REPOSITORY="$(pwd)/triton_model_repo" +TENSORRTLLM_BACKEND_DIR="/workspace/tensorrtllm_backend" +GPT_DIR="$TENSORRTLLM_BACKEND_DIR/tensorrt_llm/examples/gpt" +TOKENIZER_DIR="$GPT_DIR/gpt2" +ENGINES_DIR="${BASE_DIR}/engines/inflight_batcher_llm/${NUM_GPUS}-gpu" +TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} +SERVER=${TRITON_DIR}/bin/tritonserver +BACKEND_DIR=${TRITON_DIR}/backends +SERVER_LOG="${NAME}_server.log" +SERVER_TIMEOUT=${SERVER_TIMEOUT:=120} +CLIENT_PY=${BASE_DIR}/orca_http_test.py +CLIENT_LOG="${NAME}_orca_http_test.log" +source ../common/util.sh + +function prepare_model_repository { + rm -rf ${MODEL_REPOSITORY} && mkdir ${MODEL_REPOSITORY} + cp -r ${TENSORRTLLM_BACKEND_DIR}/all_models/inflight_batcher_llm/* ${MODEL_REPOSITORY} + rm -rf ${MODEL_REPOSITORY}/tensorrt_llm_bls + mv "${MODEL_REPOSITORY}/ensemble" "${MODEL_REPOSITORY}/${MODEL_NAME}" + + replace_config_tags "model_version: -1" "model_version: 1" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags 'name: "ensemble"' "name: \"$MODEL_NAME\"" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/${MODEL_NAME}/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${preprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/preprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${postprocessing_instance_count}' '1' "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${tokenizer_dir}' "${TOKENIZER_DIR}/" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/postprocessing/config.pbtxt" + + replace_config_tags '${triton_max_batch_size}' "128" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${decoupled_mode}' 'true' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_delay_microseconds}' "1000000" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${batching_strategy}' 'inflight_fused_batching' "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${engine_dir}' "${ENGINES_DIR}" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${triton_backend}' "tensorrtllm" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${max_queue_size}' "0" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${logits_datatype}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" + replace_config_tags '${encoder_input_features_data_type}' "TYPE_FP32" "${MODEL_REPOSITORY}/tensorrt_llm/config.pbtxt" +} + +# Wait until server health endpoint shows ready. Sets WAIT_RET to 0 on +# success, 1 on failure +function wait_for_server_ready() { + local wait_time_secs="${1:-30}" + shift + local spids=("$@") + + WAIT_RET=0 + + for _ in $(seq "$wait_time_secs"); do + for pid in "${spids[@]}"; do + if ! kill -0 "$pid" >/dev/null 2>&1; then + echo "=== Server not running." + WAIT_RET=1 + return + fi + done + + sleep 1 + + if curl -s --fail localhost:8000/v2/health/ready && + curl -s --fail -w "%{http_code}" -o /dev/null -d '{"log_verbose_level":1}' localhost:8000/v2/logging; then + return + fi + done + + echo "=== Timeout $wait_time_secs secs. Server not ready." + WAIT_RET=1 +} + +function run_server { + python3 ${TENSORRTLLM_BACKEND_DIR}/scripts/launch_triton_server.py --world_size="${NUM_GPUS}" --model_repo="${MODEL_REPOSITORY}" >${SERVER_LOG} 2>&1 & + sleep 2 # allow time to obtain the pid(s) + # Read PIDs into an array, trimming whitespaces + readarray -t SERVER_PID < <(pgrep "tritonserver") + + wait_for_server_ready ${SERVER_TIMEOUT} "${SERVER_PID[@]}" + if [ "$WAIT_RET" != "0" ]; then + # Cleanup + kill "${SERVER_PID[@]}" >/dev/null 2>&1 || true + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 + fi +} + +function kill_server { + pgrep tritonserver | xargs kill -SIGINT + for pid in "${SERVER_PID[@]}"; do + echo "Waiting for proc ${pid} to terminate..." + while kill -0 $pid >/dev/null 2>&1; do + sleep 1 + done + done +} + +clone_tensorrt_llm_backend_repo +build_gpt2_base_model +build_gpt2_tensorrt_engine +prepare_model_repository + +set +e +run_server + +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +RET=0 + +python3 $CLIENT_PY "http://localhost:8000/v2/models/${MODEL_NAME}/generate" >>$CLIENT_LOG 2>&1 + +if [ $? -ne 0 ]; then + echo "Failed: Client test had a non-zero return code." + RET=1 +fi + +if [ $RET -eq 0 ]; then + echo -e "\n***\n*** ORCA Test Passed\n***" +else + cat $SERVER_LOG + cat $CLIENT_LOG + echo -e "\n***\n*** ORCA Test FAILED\n***" +fi + +kill_server +set -e +exit $RET diff --git a/qa/L0_output_name/output_name_test.py b/qa/L0_output_name/output_name_test.py index 905174640c..8affb2e1e1 100755 --- a/qa/L0_output_name/output_name_test.py +++ b/qa/L0_output_name/output_name_test.py @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -36,7 +36,7 @@ import grpc -_trials = ("graphdef", "libtorch", "onnx", "plan", "savedmodel") +_trials = ("libtorch", "onnx", "plan") class OutputNameValidationTest(tu.TestResultCollector): diff --git a/qa/L0_perf_deeprecommender/run_test.sh b/qa/L0_perf_deeprecommender/run_test.sh index 75fd68704d..434803e7a9 100755 --- a/qa/L0_perf_deeprecommender/run_test.sh +++ b/qa/L0_perf_deeprecommender/run_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,13 +28,12 @@ STATIC_BATCH_SIZES=${STATIC_BATCH_SIZES:=1} DYNAMIC_BATCH_SIZES=${DYNAMIC_BATCH_SIZES:=1} INSTANCE_COUNTS=${INSTANCE_COUNTS:=1} -TF_VERSION=${TF_VERSION:=2} PERF_CLIENT=../clients/perf_client REPORTER=../common/reporter.py SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models --backend-config=tensorflow,version=${TF_VERSION}" +SERVER_ARGS="--model-repository=`pwd`/models" source ../common/util.sh # Select the single GPU that will be available to the inference diff --git a/qa/L0_perf_nomodel/run_test.sh b/qa/L0_perf_nomodel/run_test.sh index ce3350d97b..fcfdf16e0e 100755 --- a/qa/L0_perf_nomodel/run_test.sh +++ b/qa/L0_perf_nomodel/run_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -27,7 +27,7 @@ REPO_VERSION=$1 -BACKENDS=${BACKENDS:="plan custom graphdef savedmodel onnx libtorch python"} +BACKENDS=${BACKENDS:="plan custom onnx libtorch python"} STATIC_BATCH_SIZES=${STATIC_BATCH_SIZES:=1} DYNAMIC_BATCH_SIZES=${DYNAMIC_BATCH_SIZES:=1} INSTANCE_COUNTS=${INSTANCE_COUNTS:=1} @@ -50,8 +50,7 @@ SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends MODEL_REPO="${PWD}/models" PERF_CLIENT=../clients/perf_client -TF_VERSION=${TF_VERSION:=2} -SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR} --backend-config=tensorflow,version=${TF_VERSION}" +SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR}" source ../common/util.sh # DATADIR is already set in environment variable for aarch64 diff --git a/qa/L0_perf_nomodel/test.sh b/qa/L0_perf_nomodel/test.sh index a213d24e9d..c9a4ece9aa 100755 --- a/qa/L0_perf_nomodel/test.sh +++ b/qa/L0_perf_nomodel/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -168,7 +168,7 @@ TEST_CONCURRENCY+=( 16 16 16) -TEST_BACKENDS=${BACKENDS:="plan custom graphdef savedmodel onnx libtorch python"} +TEST_BACKENDS=${BACKENDS:="plan custom onnx libtorch python"} mkdir -p ${REPO_VERSION} diff --git a/qa/L0_perf_pyclients/test.sh b/qa/L0_perf_pyclients/test.sh index 9b7e405977..a0b4def392 100755 --- a/qa/L0_perf_pyclients/test.sh +++ b/qa/L0_perf_pyclients/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -43,10 +43,8 @@ REPORTER=../common/reporter.py CLIENT_LOG="./simple_perf_client.log" SIMPLE_PERF_CLIENT=simple_perf_client.py -TF_VERSION=${TF_VERSION:=2} - SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/custom_models --backend-config=tensorflow,version=${TF_VERSION}" +SERVER_ARGS="--model-repository=`pwd`/custom_models" source ../common/util.sh # Select the single GPU that will be available to the inference diff --git a/qa/L0_perf_resnet/run_test.sh b/qa/L0_perf_resnet/run_test.sh index 579d00c0e5..3e7b048c40 100755 --- a/qa/L0_perf_resnet/run_test.sh +++ b/qa/L0_perf_resnet/run_test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -28,7 +28,6 @@ STATIC_BATCH=${STATIC_BATCH:=1} INSTANCE_CNT=${INSTANCE_CNT:=1} BACKEND_CONFIG=${BACKEND_CONFIG:=""} -TF_VERSION=${TF_VERSION:=2} REPORTER=../common/reporter.py @@ -36,7 +35,7 @@ TRITON_DIR=${TRITON_DIR:="/opt/tritonserver"} SERVER=${TRITON_DIR}/bin/tritonserver BACKEND_DIR=${TRITON_DIR}/backends MODEL_REPO="${PWD}/models" -SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR} ${BACKEND_CONFIG} --backend-config=tensorflow,version=${TF_VERSION}" +SERVER_ARGS="--model-repository=${MODEL_REPO} --backend-directory=${BACKEND_DIR} ${BACKEND_CONFIG}" source ../common/util.sh # Select the single GPU that will be available to the inference diff --git a/qa/L0_perf_resnet/test.sh b/qa/L0_perf_resnet/test.sh index 35d5b174be..11538db4ce 100755 --- a/qa/L0_perf_resnet/test.sh +++ b/qa/L0_perf_resnet/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -234,6 +234,5 @@ if [ "$ARCH" == "x86_64" ]; then INSTANCE_CNT=${INSTANCE_CNT} \ CONCURRENCY=${CONCURRENCY} \ ARCH=${ARCH} \ - BACKEND_CONFIG=" --backend-config=tensorflow,version=2" \ bash -x run_test.sh fi diff --git a/qa/L0_response_cache/ensemble_cache_test.py b/qa/L0_response_cache/ensemble_cache_test.py index 96b959cc8e..36821d07e1 100755 --- a/qa/L0_response_cache/ensemble_cache_test.py +++ b/qa/L0_response_cache/ensemble_cache_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -48,8 +48,8 @@ def setUp(self): self.triton_client = grpcclient.InferenceServerClient( "localhost:8001", verbose=True ) - self.ensemble_model = "simple_graphdef_float32_float32_float32" - self.composing_model = "graphdef_float32_float32_float32" + self.ensemble_model = "simple_onnx_float32_float32_float32" + self.composing_model = "onnx_float32_float32_float32" self.model_directory = os.path.join(os.getcwd(), "models", "ensemble_models") self.ensemble_config_file = os.path.join( self.model_directory, self.ensemble_model, "config.pbtxt" @@ -79,6 +79,15 @@ def _update_config(self, config_file, config_pattern, config_to_add): config_data += config_to_add f.write(config_data) + def _add_instance_group_cpu(self, config_file): + # Utility function to add instance group of kind CPU to the config file + with open(config_file, "r") as f: + config_data = f.read() + if "instance_group" not in config_data: + with open(config_file, "w") as f: + config_data += "instance_group {\n kind: KIND_CPU\n}\n" + f.write(config_data) + def _remove_config(self, config_file, config_to_remove): # Utility function to remove extra added config from the config files with open(config_file, "r") as f: @@ -125,7 +134,7 @@ def _run_inference_and_validate(self, model): Helper function that takes model as a parameter to verify the corresponding model's stats The passed model is composing model for test case `test_ensemble_composing_model_cache_enabled` For other testcases, the top-level ensemble model stats are verified. - * loads the simple_graphdef_float32_float32_float32 and graphdef_float32_float32_float32 + * loads the simple_onnx_float32_float32_float32 and onnx_float32_float32_float32 and verifies if they are loaded properly. * Checks the initial statistics of the model passed in the parameter Expected - baseline statistics to be all empty metrics since @@ -252,6 +261,8 @@ def test_ensemble_composing_model_cache_enabled(self): self._update_config( self.composing_config_file, RESPONSE_CACHE_PATTERN, RESPONSE_CACHE_CONFIG ) + # Currently, response cache is supported only for tensors on CPU. + self._add_instance_group_cpu(self.composing_config_file) self._run_inference_and_validate(self.composing_model) ensemble_model_stats = self._get_model_statistics(self.ensemble_model) composing_model_stats = self._get_model_statistics(self.composing_model) diff --git a/qa/L0_response_cache/test.sh b/qa/L0_response_cache/test.sh index be78059d1e..c99aac89d1 100755 --- a/qa/L0_response_cache/test.sh +++ b/qa/L0_response_cache/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -64,8 +64,8 @@ ENSEMBLE_CACHE_COMPOSING_DECOUPLED="${MODEL_DIR}/ensemble_cache_composing_decoup rm -fr ${ENSEMBLE_MODEL_DIR} && mkdir ${ENSEMBLE_MODEL_DIR} rm -fr ${ENSEMBLE_CACHE_DECOUPLED} && mkdir ${ENSEMBLE_CACHE_DECOUPLED} rm -fr ${ENSEMBLE_CACHE_COMPOSING_DECOUPLED} && mkdir ${ENSEMBLE_CACHE_COMPOSING_DECOUPLED} -ENSEMBLE_MODEL="simple_graphdef_float32_float32_float32" -COMPOSING_MODEL="graphdef_float32_float32_float32" +ENSEMBLE_MODEL="simple_onnx_float32_float32_float32" +COMPOSING_MODEL="onnx_float32_float32_float32" cp -r "/data/inferenceserver/${REPO_VERSION}/qa_ensemble_model_repository/qa_model_repository/${ENSEMBLE_MODEL}" "${ENSEMBLE_MODEL_DIR}/${ENSEMBLE_MODEL}" cp -r "/data/inferenceserver/${REPO_VERSION}/qa_model_repository/${COMPOSING_MODEL}" "${ENSEMBLE_MODEL_DIR}/${COMPOSING_MODEL}" diff --git a/qa/L0_savedmodel_shape/saved_model_shape_test.py b/qa/L0_savedmodel_shape/saved_model_shape_test.py deleted file mode 100755 index b5ae13a680..0000000000 --- a/qa/L0_savedmodel_shape/saved_model_shape_test.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import infer_util as iu -import numpy as np -import test_util as tu - -np_dtype_string = np.dtype(object) - - -class SavedModelShapeTest(tu.TestResultCollector): - def _full_exact( - self, input_dtype, output0_dtype, output1_dtype, output0_raw, output1_raw, swap - ): - def _infer_exact_helper( - tester, - pf, - tensor_shape, - batch_size, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=True, - output1_raw=True, - model_version=None, - swap=False, - outputs=("OUTPUT0", "OUTPUT1"), - use_http=True, - use_grpc=True, - skip_request_id_check=False, - use_streaming=True, - correlation_id=0, - ): - for bs in (1, batch_size): - # model that does not support batching - if bs == 1: - iu.infer_exact( - tester, - "savedmodel_nobatch", - tensor_shape, - bs, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - model_version=model_version, - swap=swap, - outputs=outputs, - use_http=use_http, - use_grpc=use_grpc, - skip_request_id_check=skip_request_id_check, - use_streaming=use_streaming, - correlation_id=correlation_id, - ) - # model that supports batching - iu.infer_exact( - tester, - "savedmodel", - (bs,) + tensor_shape, - bs, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - model_version=model_version, - swap=swap, - outputs=outputs, - use_http=use_http, - use_grpc=use_grpc, - skip_request_id_check=skip_request_id_check, - use_streaming=use_streaming, - correlation_id=correlation_id, - ) - - input_size = 16 - - if tu.validate_for_tf_model( - input_dtype, - output0_dtype, - output1_dtype, - (input_size,), - (input_size,), - (input_size,), - ): - _infer_exact_helper( - self, - "savedmodel", - (input_size,), - 8, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - swap=swap, - ) - - def test_raw_bbb(self): - self._full_exact( - np.int8, np.int8, np.int8, output0_raw=True, output1_raw=True, swap=True - ) - - def test_raw_sss(self): - self._full_exact( - np.int16, np.int16, np.int16, output0_raw=True, output1_raw=True, swap=True - ) - - def test_raw_iii(self): - self._full_exact( - np.int32, np.int32, np.int32, output0_raw=True, output1_raw=True, swap=True - ) - - def test_raw_lll(self): - self._full_exact( - np.int64, np.int64, np.int64, output0_raw=True, output1_raw=True, swap=False - ) - - def test_raw_hhh(self): - self._full_exact( - np.float16, - np.float16, - np.float16, - output0_raw=True, - output1_raw=True, - swap=False, - ) - - def test_raw_fff(self): - self._full_exact( - np.float32, - np.float32, - np.float32, - output0_raw=True, - output1_raw=True, - swap=True, - ) - - def test_raw_hff(self): - self._full_exact( - np.float16, - np.float32, - np.float32, - output0_raw=True, - output1_raw=True, - swap=False, - ) - - def test_raw_bii(self): - self._full_exact( - np.int8, np.int32, np.int32, output0_raw=True, output1_raw=True, swap=False - ) - - def test_raw_ibb(self): - self._full_exact( - np.int32, np.int8, np.int8, output0_raw=True, output1_raw=True, swap=False - ) - - def test_raw_ibs(self): - self._full_exact( - np.int32, np.int8, np.int16, output0_raw=True, output1_raw=True, swap=False - ) - - def test_raw_iff(self): - self._full_exact( - np.int32, - np.float32, - np.float32, - output0_raw=True, - output1_raw=True, - swap=False, - ) - - def test_raw_fii(self): - self._full_exact( - np.float32, - np.int32, - np.int32, - output0_raw=True, - output1_raw=True, - swap=False, - ) - - def test_raw_ihs(self): - self._full_exact( - np.int32, - np.float16, - np.int16, - output0_raw=True, - output1_raw=True, - swap=False, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_savedmodel_shape/test.sh b/qa/L0_savedmodel_shape/test.sh deleted file mode 100755 index e059a5bf0b..0000000000 --- a/qa/L0_savedmodel_shape/test.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -TEST_RESULT_FILE='test_results.txt' -CLIENT_LOG_BASE="./client_saved_model_shape" -INFER_TEST=saved_model_shape_test.py -EXPECTED_NUM_TESTS="13" - -DATADIR=`pwd`/models - -SERVER=/opt/tritonserver/bin/tritonserver -# Allow more time to exit. Ensemble brings in too many models -SERVER_ARGS="--model-repository=$DATADIR --exit-timeout-secs=120" -SERVER_LOG_BASE="./server_saved_model_shape" -source ../common/util.sh - -rm -f $SERVER_LOG_BASE* $CLIENT_LOG_BASE* - -RET=0 - -SERVER_LOG=$SERVER_LOG_BASE.${TARGET}.log -CLIENT_LOG=$CLIENT_LOG_BASE.${TARGET}.log - -rm -fr models && \ - cp -r /data/inferenceserver/${REPO_VERSION}/qa_noshape_model_repository models - -create_nop_version_dir `pwd`/models - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -# python unittest seems to swallow ImportError and still return 0 -# exit code. So need to explicitly check CLIENT_LOG to make sure -# we see some running tests -python $INFER_TEST >$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -else - check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi - -set -e - -kill $SERVER_PID -wait $SERVER_PID - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -fi - -exit $RET diff --git a/qa/L0_sequence_batcher/sequence_batcher_test.py b/qa/L0_sequence_batcher/sequence_batcher_test.py index 3e6cfc032a..d6d576da5a 100755 --- a/qa/L0_sequence_batcher/sequence_batcher_test.py +++ b/qa/L0_sequence_batcher/sequence_batcher_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -57,7 +57,7 @@ else: _protocols = ("http",) -BACKENDS = os.environ.get("BACKENDS", "graphdef savedmodel onnx plan custom python") +BACKENDS = os.environ.get("BACKENDS", "onnx plan custom python") ENSEMBLES = bool(int(os.environ.get("ENSEMBLES", 1))) NO_BATCHING = int(os.environ["NO_BATCHING"]) == 1 @@ -126,10 +126,6 @@ def get_datatype(self, trial): return (np.float32,) if "custom" in trial: return (np.int32,) - if "savedmodel" in trial: - return (np.float32, np.bool_) - if "graphdef" in trial: - return (np.dtype(object), np.bool_) # Only test the string data type for ONNX and libtorch models in implicit state if IMPLICIT_STATE: @@ -148,7 +144,6 @@ def get_expected_result(self, expected_result, value, trial, flag_str=None): # information. if ( (not NO_BATCHING and ("custom" not in trial)) - or ("graphdef" in trial) or ("plan" in trial) or ("onnx" in trial) ) or ("libtorch" in trial): diff --git a/qa/L0_sequence_batcher/test.sh b/qa/L0_sequence_batcher/test.sh index ac34458b4e..229a935319 100755 --- a/qa/L0_sequence_batcher/test.sh +++ b/qa/L0_sequence_batcher/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -87,8 +87,6 @@ if [ "$TEST_JETSON" -eq 1 ]; then MODEL_TRIALS="0 v" fi -TF_VERSION=${TF_VERSION:=2} - # On windows the paths invoked by the script (running in WSL) must use # /mnt/c when needed but the paths on the tritonserver command-line # must be C:/ style. @@ -116,14 +114,14 @@ else fi fi -SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR} --backend-config=tensorflow,version=${TF_VERSION} --log-verbose=1" +SERVER_ARGS_EXTRA="--backend-directory=${BACKEND_DIR} --log-verbose=1" source ../common/util.sh RET=0 # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx plan libtorch custom python"} +BACKENDS=${BACKENDS:="onnx plan libtorch custom python"} export BACKENDS # If MODEL_TRIALS not specified set to 0 1 2 4 v @@ -198,10 +196,6 @@ function get_datatype () { local dtype="int32 bool" if [[ $1 == "plan" ]]; then dtype="float32" - elif [[ $1 == "savedmodel" ]]; then - dtype="float32 bool" - elif [[ $1 == "graphdef" ]]; then - dtype="object bool int32" fi # Add type string to the onnx model tests only for implicit state. diff --git a/qa/L0_sequence_corrid_batcher/sequence_corrid_batcher_test.py b/qa/L0_sequence_corrid_batcher/sequence_corrid_batcher_test.py index c9883c9133..956b9241d0 100755 --- a/qa/L0_sequence_corrid_batcher/sequence_corrid_batcher_test.py +++ b/qa/L0_sequence_corrid_batcher/sequence_corrid_batcher_test.py @@ -48,9 +48,9 @@ _model_instances = int(os.environ["MODEL_INSTANCES"]) if _no_batching: - _trials = ("savedmodel_nobatch", "graphdef_nobatch", "plan_nobatch", "onnx_nobatch") + _trials = ("plan_nobatch", "onnx_nobatch") else: - _trials = ("savedmodel", "graphdef", "plan", "onnx") + _trials = ("plan", "onnx") _protocols = ("http", "grpc") _max_sequence_idle_ms = 5000 @@ -67,7 +67,6 @@ def get_expected_result(self, expected_result, corrid, value, trial, flag_str=No # information. if ( (("nobatch" not in trial) and ("custom" not in trial)) - or ("graphdef" in trial) or ("plan" in trial) or ("onnx" in trial) ) or ("libtorch" in trial): diff --git a/qa/L0_sequence_corrid_batcher/test.sh b/qa/L0_sequence_corrid_batcher/test.sh index 3948cd7445..e876419b9b 100755 --- a/qa/L0_sequence_corrid_batcher/test.sh +++ b/qa/L0_sequence_corrid_batcher/test.sh @@ -59,8 +59,6 @@ export CUDA_VISIBLE_DEVICES=0 # models4 - four instances with batch-size 1 rm -fr *.log models{0,1,2,4} && mkdir models4 for m in \ - $DATADIR/qa_dyna_sequence_model_repository/graphdef_dyna_sequence_int32 \ - $DATADIR/qa_dyna_sequence_model_repository/savedmodel_dyna_sequence_int32 \ $DATADIR/qa_dyna_sequence_model_repository/plan_dyna_sequence_int32 \ $DATADIR/qa_dyna_sequence_model_repository/onnx_dyna_sequence_int32 \ $DATADIR/qa_dyna_sequence_model_repository/libtorch_dyna_sequence_int32; do diff --git a/qa/L0_sequence_stress/sequence_stress.py b/qa/L0_sequence_stress/sequence_stress.py index bd71e9bcc2..c8b14cf90b 100755 --- a/qa/L0_sequence_stress/sequence_stress.py +++ b/qa/L0_sequence_stress/sequence_stress.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -87,12 +87,7 @@ def check_sequence_async( (flag_str, value, expected_result, delay_ms) """ - if ( - ("savedmodel" in trial) - or ("graphdef" in trial) - or ("custom" in trial) - or ("plan" in trial) - ): + if ("custom" in trial) or ("plan" in trial): tensor_shape = ( 1, 1, @@ -176,10 +171,8 @@ def check_sequence_async( def get_datatype(trial): # Get the datatype to use based on what models are available (see test.sh) - if ("plan" in trial) or ("savedmodel" in trial): + if "plan" in trial: return np.float32 - if "graphdef" in trial: - return np.dtype(object) return np.int32 diff --git a/qa/L0_server_status/server_status_test.py b/qa/L0_server_status/server_status_test.py index 53a091e046..9a0f26e7b9 100755 --- a/qa/L0_server_status/server_status_test.py +++ b/qa/L0_server_status/server_status_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,7 +45,7 @@ class ServerMetadataTest(tu.TestResultCollector): def test_basic(self): try: for pair in [("localhost:8000", "http"), ("localhost:8001", "grpc")]: - model_name = "graphdef_int32_int8_int8" + model_name = "libtorch_int32_int8_int8" extensions = [ "classification", "sequence", @@ -129,7 +129,7 @@ def test_unknown_model(self): def test_unknown_model_version(self): try: for pair in [("localhost:8000", "http"), ("localhost:8001", "grpc")]: - model_name = "graphdef_int32_int8_int8" + model_name = "onnx_int32_int8_int8" if pair[1] == "http": triton_client = httpclient.InferenceServerClient( url=pair[0], verbose=True @@ -149,18 +149,18 @@ def test_unknown_model_version(self): except InferenceServerException as ex: self.assertTrue( ex.message().startswith( - "Request for unknown model: 'graphdef_int32_int8_int8' version 99 is not found" + "Request for unknown model: 'onnx_int32_int8_int8' version 99 is not found" ) ) def test_model_latest_infer(self): input_size = 16 tensor_shape = (1, input_size) - platform_name = {"graphdef": "tensorflow_graphdef", "onnx": "onnxruntime_onnx"} + platform_name = {"plan": "tensorrt_plan", "onnx": "onnxruntime_onnx"} # There are 3 versions of *_int32_int32_int32 and all # should be available. - for platform in ("graphdef", "onnx"): + for platform in ("plan", "onnx"): model_name = platform + "_int32_int32_int32" # Initially there should be no version stats.. @@ -316,7 +316,7 @@ def test_model_specific_infer(self): # There are 3 versions of *_float32_float32_float32 but only # versions 1 and 3 should be available. - for platform in ("graphdef", "onnx", "plan"): + for platform in ("libtorch", "onnx", "plan"): tensor_shape = (1, input_size) model_name = platform + "_float32_float32_float32" @@ -439,7 +439,7 @@ def test_model_versions_deleted(self): # version 3 was executed once. Version 2 and 3 models were # deleted from the model repository so now only expect version 1 to # be ready and show stats. - for platform in ("graphdef", "onnx"): + for platform in ("libtorch", "onnx"): model_name = platform + "_int32_int32_int32" try: @@ -513,7 +513,7 @@ def test_model_versions_added(self): # Originally There was version 1 of *_float16_float32_float32. # Version 7 was added so now expect just version 7 to be ready # and provide infer stats. - for platform in ("graphdef",): + for platform in ("plan",): model_name = platform + "_float16_float32_float32" try: @@ -615,7 +615,7 @@ def test_infer_stats_no_model_version(self): # version 3 was executed once. Version 2 and 3 models were # deleted from the model repository so now only expect version 1 to # be ready and show infer stats. - for platform in ("graphdef", "onnx"): + for platform in ("libtorch", "onnx"): model_name = platform + "_int32_int32_int32" try: @@ -723,8 +723,8 @@ def test_infer_stats_no_model(self): stats = infer_stats.model_stats self.assertEqual( len(stats), - 221, - "expected 221 infer stats for all ready versions of all model", + 125, + "expected 125 infer stats for all ready versions of all model", ) except InferenceServerException as ex: diff --git a/qa/L0_server_status/test.sh b/qa/L0_server_status/test.sh index d66b4d5a44..7d896f48fb 100755 --- a/qa/L0_server_status/test.sh +++ b/qa/L0_server_status/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -84,9 +84,9 @@ fi set -e -rm -fr models/graphdef_int32_int32_int32/2 models/graphdef_int32_int32_int32/3 +rm -fr models/libtorch_int32_int32_int32/2 models/libtorch_int32_int32_int32/3 rm -fr models/onnx_int32_int32_int32/2 models/onnx_int32_int32_int32/3 -cp -r models/graphdef_float16_float32_float32/1 models/graphdef_float16_float32_float32/7 +cp -r models/plan_float16_float32_float32/1 models/plan_float16_float32_float32/7 sleep 3 # Dumping the contents of the models that are currently loaded for debugging purposes diff --git a/qa/L0_simple_ensemble/models/simple/config.pbtxt b/qa/L0_simple_ensemble/models/simple/config.pbtxt index 7e7a178fc1..d5627d3db3 100644 --- a/qa/L0_simple_ensemble/models/simple/config.pbtxt +++ b/qa/L0_simple_ensemble/models/simple/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 version_policy: { all {} } input [ diff --git a/qa/L0_simple_lib/test.sh b/qa/L0_simple_lib/test.sh index 7045f512ef..714f61f752 100755 --- a/qa/L0_simple_lib/test.sh +++ b/qa/L0_simple_lib/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,7 +55,7 @@ for SIMPLE_CLIENT in simple ; do CLIENT_LOG=$SIMPLE_CLIENT SIMPLE_CLIENT=./$SIMPLE_CLIENT - for trial in graphdef savedmodel onnx libtorch plan; do + for trial in onnx libtorch plan; do full=${trial}_float32_float32_float32 rm -rf $MODELSDIR mkdir -p $MODELSDIR/simple/1 && \ @@ -88,10 +88,10 @@ for SIMPLE_CLIENT in simple ; do set -e done - # Use savedmodel for addsub ensemble + # Use onnx for addsub ensemble mkdir -p $MODELSDIR/simple/1 - cp -r $DATADIR/savedmodel_float32_float32_float32/1/* $MODELSDIR/simple/1/. - cp $DATADIR/savedmodel_float32_float32_float32/config.pbtxt $MODELSDIR/simple/. + cp -r $DATADIR/onnx_float32_float32_float32/1/* $MODELSDIR/simple/1/. + cp $DATADIR/onnx_float32_float32_float32/config.pbtxt $MODELSDIR/simple/. (cd $MODELSDIR/simple && \ sed -i "s/^name:.*/name: \"simple\"/" config.pbtxt && \ sed -i "s/label_filename:.*//" config.pbtxt) diff --git a/qa/L0_socket/models/simple/config.pbtxt b/qa/L0_socket/models/simple/config.pbtxt index 838edd5d55..dd82914e26 100644 --- a/qa/L0_socket/models/simple/config.pbtxt +++ b/qa/L0_socket/models/simple/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,7 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "simple" -platform: "tensorflow_graphdef" +platform: "onnxruntime_onnx" max_batch_size: 8 input [ { diff --git a/qa/L0_storage_S3/test.sh b/qa/L0_storage_S3/test.sh index 9f2b67cef4..143b78656e 100755 --- a/qa/L0_storage_S3/test.sh +++ b/qa/L0_storage_S3/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,7 +45,7 @@ CLIENT_LOG_BASE="./client" INFER_TEST="../common/infer_test.py" EXPECTED_NUM_TESTS="3" TEST_RESULT_FILE='test_results.txt' -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan"} +BACKENDS=${BACKENDS:="onnx libtorch plan"} # S3 credentials are necessary for this test. Pass via ENV variables aws configure set default.region $AWS_DEFAULT_REGION && \ diff --git a/qa/L0_storage_S3_local/test.sh b/qa/L0_storage_S3_local/test.sh index e60b106b31..3add65df60 100755 --- a/qa/L0_storage_S3_local/test.sh +++ b/qa/L0_storage_S3_local/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,7 @@ EXPECTED_NUM_TESTS="3" DATADIR="/data/inferenceserver/${REPO_VERSION}/qa_model_repository" # Used to control which backends are run in infer_test.py -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan"} +BACKENDS=${BACKENDS:="onnx libtorch plan"} function run_unit_tests() { echo "Running unit tests: ${INFER_TEST}" @@ -228,7 +228,7 @@ awslocal $ENDPOINT_FLAG s3 rm s3://demo-bucket1.0 --recursive --include "*" && \ # Test with Polling, no model configuration file - with strict model config disabled echo "=== Running autocomplete tests ===" -AUTOCOMPLETE_BACKENDS="savedmodel" +AUTOCOMPLETE_BACKENDS="onnx" export BACKENDS=${AUTOCOMPLETE_BACKENDS} set +e @@ -241,9 +241,8 @@ for BACKEND in ${AUTOCOMPLETE_BACKENDS}; do # Config files specify things expected by unit test like label_filename # and max_batch_size for comparing results, so remove some key fields # for autocomplete to fill that won't break the unit test. - sed -i '/platform:/d' models/${model}/config.pbtxt - sed -i '/data_type:/d' models/${model}/config.pbtxt - sed -i '/dims:/d' models/${model}/config.pbtxt + sed -i '/^input {/,/^}/d' models/${model}/config.pbtxt + sed -i '/^output {/,/^}/d' models/${model}/config.pbtxt done done set -e @@ -275,8 +274,8 @@ awslocal $ENDPOINT_FLAG s3 rm s3://demo-bucket1.0 --recursive --include "*" && \ # Test for multiple model repositories using S3 cloud storage echo "=== Running multiple-model-repository tests ===" -BACKENDS1="graphdef libtorch" -BACKENDS2="onnx plan savedmodel" +BACKENDS1="libtorch" +BACKENDS2="onnx plan" export BACKENDS="$BACKENDS1 $BACKENDS2" set +e diff --git a/qa/L0_storage_azure/test.sh b/qa/L0_storage_azure/test.sh index d4b367780a..805c081679 100755 --- a/qa/L0_storage_azure/test.sh +++ b/qa/L0_storage_azure/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -81,7 +81,7 @@ rm -f $SERVER_LOG_BASE* $CLIENT_LOG_BASE* RET=0 # Used to control which backends are run in infer_test.py -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan"} +BACKENDS=${BACKENDS:="onnx libtorch plan"} function run_unit_tests() { BACKENDS=$BACKENDS python $INFER_TEST >$CLIENT_LOG 2>&1 @@ -252,16 +252,15 @@ sleep 10 # Setup model repository with minimal configs to be autocompleted rm -rf models && mkdir -p models -AUTOCOMPLETE_BACKENDS="savedmodel" +AUTOCOMPLETE_BACKENDS="onnx" for FW in ${AUTOCOMPLETE_BACKENDS}; do for model in ${FW}_float32_float32_float32 ${FW}_object_object_object; do cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/${model} models/ # Config files specify things expected by unit test like label_filename # and max_batch_size for comparing results, so remove some key fields # for autocomplete to fill that won't break the unit test. - sed -i '/platform:/d' models/${model}/config.pbtxt - sed -i '/data_type:/d' models/${model}/config.pbtxt - sed -i '/dims:/d' models/${model}/config.pbtxt + sed -i '/^input {/,/^}/d' models/${model}/config.pbtxt + sed -i '/^output {/,/^}/d' models/${model}/config.pbtxt done done diff --git a/qa/L0_storage_swiftstack/infer_test.py b/qa/L0_storage_swiftstack/infer_test.py index f8a65a01a4..922caeb900 100755 --- a/qa/L0_storage_swiftstack/infer_test.py +++ b/qa/L0_storage_swiftstack/infer_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -83,28 +83,6 @@ def _infer_exact_helper( input_size = 16 - if tu.validate_for_tf_model( - input_dtype, - output0_dtype, - output1_dtype, - (input_size,), - (input_size,), - (input_size,), - ): - for pf in ["graphdef", "savedmodel"]: - _infer_exact_helper( - self, - pf, - (input_size,), - 8, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - swap=swap, - ) - if tu.validate_for_trt_model( input_dtype, output0_dtype, diff --git a/qa/L0_storage_swiftstack/test.sh b/qa/L0_storage_swiftstack/test.sh index 99fb5610d6..e8c0edf477 100755 --- a/qa/L0_storage_swiftstack/test.sh +++ b/qa/L0_storage_swiftstack/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -104,11 +104,11 @@ aws s3 rm $BUCKET_URL/ --recursive --include "*" # Now start model tests -for FW in graphdef savedmodel onnx libtorch plan; do +for FW in onnx libtorch plan; do cp -r /data/inferenceserver/${REPO_VERSION}/qa_model_repository/${FW}_float32_float32_float32/ models/ done -for FW in graphdef savedmodel onnx libtorch plan; do +for FW in onnx libtorch plan; do for MC in `ls models/${FW}*/config.pbtxt`; do echo "instance_group [ { kind: KIND_GPU }]" >> $MC done diff --git a/qa/L0_string_io/string_client_test.py b/qa/L0_string_io/string_client_test.py index 16112ac70c..1f44c9913a 100755 --- a/qa/L0_string_io/string_client_test.py +++ b/qa/L0_string_io/string_client_test.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -199,8 +199,7 @@ def _test_bytes(self, model_name): with self.assertRaises(tritonutils.InferenceServerException): self._test_unicode_bytes_dtype(client, model_name, dtype) - def test_tf_unicode_bytes(self): - self._test_bytes("graphdef_nobatch_zero_1_object") + def test_unicode_bytes(self): self._test_bytes("string_identity") diff --git a/qa/L0_string_io/test.sh b/qa/L0_string_io/test.sh index eb45d43ba2..538b1f7e90 100755 --- a/qa/L0_string_io/test.sh +++ b/qa/L0_string_io/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -52,8 +52,8 @@ source ../common/util.sh rm -f $CLIENT_LOG $SERVER_LOG rm -fr models && mkdir models -cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/graphdef_nobatch_zero_1_object models/. -cp -r ../python_models/string_identity models/. +cp -rv /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/onnx_zero_1_object/ models/. +cp -rv ../python_models/string_identity models/. mkdir models/string_identity/1/ mv models/string_identity/model.py models/string_identity/1/model.py diff --git a/qa/L0_tf_gpu_io/test.sh b/qa/L0_tf_gpu_io/test.sh deleted file mode 100755 index 98a5dff1ef..0000000000 --- a/qa/L0_tf_gpu_io/test.sh +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -TF_TEST=tf_gpu_io_test.py -BACKENDS=${BACKENDS:="graphdef savedmodel"} - -DATADIR=/data/inferenceserver/${REPO_VERSION} - -SERVER=/opt/tritonserver/bin/tritonserver -source ../common/util.sh - -RET=0 -rm -f ./*.log - -# Test with qa identity TF models -for BACKEND in $BACKENDS; do - MODEL_NAME=${BACKEND}_zero_1_float32 - rm -fr models && mkdir -p models - cp -r $DATADIR/qa_identity_model_repository/${MODEL_NAME} \ - models/${MODEL_NAME}_def && \ - (cd models/${MODEL_NAME}_def && \ - sed -i 's/_zero_1_float32/&_def/' config.pbtxt) && \ - # Enable GPU I/O for TensorFlow model - cp -r models/${MODEL_NAME}_def models/${MODEL_NAME}_gpu && \ - (cd models/${MODEL_NAME}_gpu && \ - sed -i 's/_zero_1_float32_def/_zero_1_float32_gpu/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"gpu_io\"} ] } }" >> config.pbtxt) - - SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" - SERVER_LOG="${MODEL_NAME}.server.log" - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - - python $TF_TEST TfGpuIoTest.test_${MODEL_NAME}_def >> ${BACKEND}.sanity.log 2>&1 - if (( $? != 0 )); then - cat ${BACKEND}.sanity.log - RET=1 - fi - - grep "is GPU tensor: true" $SERVER_LOG >> grep.out.log - if [ $? -eq 0 ]; then - echo -e "\n***\n*** Failed. Expected neither input or output is GPU tensor\n***" - RET=1 - fi - - python $TF_TEST TfGpuIoTest.test_${MODEL_NAME}_gpu >> ${BACKEND}.gpu.sanity.log 2>&1 - if (( $? != 0 )); then - cat ${BACKEND}.gpu.sanity.log - RET=1 - fi - - grep "is GPU tensor: true" $SERVER_LOG >> grep.out.log - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected input and output are GPU tensors\n***" - RET=1 - fi - - set -e - - kill $SERVER_PID - wait $SERVER_PID -done - -# Test savedmodel with mismatched key and name -rm -rf models && mkdir -p models -cp -r $DATADIR/qa_tf_tag_sigdef_repository/sig_tag0 models -(cd models/sig_tag0 && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"gpu_io\"} ] } }" >> config.pbtxt) - -SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1" -SERVER_LOG="sig_tag0.server.log" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -CLIENT_LOG="sig_tag0.gpu.log" -python $TF_TEST TfGpuIoTest.test_sig_tag0 >> $CLIENT_LOG 2>&1 -if (( $? != 0 )); then - cat $CLIENT_LOG - RET=1 -fi -grep "is GPU tensor: true" $SERVER_LOG >> grep.out.log -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected input and output are GPU tensors\n***" - RET=1 -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - echo -e "\n***\n*** Test FAILED\n***" -fi -exit $RET diff --git a/qa/L0_tf_gpu_io/tf_gpu_io_test.py b/qa/L0_tf_gpu_io/tf_gpu_io_test.py deleted file mode 100755 index fd3550e434..0000000000 --- a/qa/L0_tf_gpu_io/tf_gpu_io_test.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import infer_util as iu -import numpy as np -import test_util as tu - -TENSOR_SIZE = 16384 - - -class TfGpuIoTest(tu.TestResultCollector): - def _test_helper( - self, - model_name, - shape, - override_input_names=[], - override_output_names=[], - batching_enabled=False, - ): - try: - bs = 1 - if batching_enabled: - shape = [ - [ - bs, - ] - + shape - ] - iu.infer_zero( - self, - "graphdef", - bs, - np.float32, - shape, - shape, - override_model_name=model_name, - override_input_names=override_input_names, - override_output_names=override_output_names, - ) - - except Exception as ex: - self.assertTrue(False, "unexpected error {}".format(ex)) - - def test_sig_tag0(self): - self._test_helper( - "sig_tag0", - [16], - override_input_names=["INPUT"], - override_output_names=["OUTPUT"], - ) - - def test_graphdef_zero_1_float32_def(self): - self._test_helper( - "graphdef_zero_1_float32_def", [TENSOR_SIZE], batching_enabled=True - ) - - def test_graphdef_zero_1_float32_gpu(self): - self._test_helper( - "graphdef_zero_1_float32_gpu", [TENSOR_SIZE], batching_enabled=True - ) - - def test_savedmodel_zero_1_float32_def(self): - self._test_helper( - "savedmodel_zero_1_float32_def", [TENSOR_SIZE], batching_enabled=True - ) - - def test_savedmodel_zero_1_float32_gpu(self): - self._test_helper( - "savedmodel_zero_1_float32_gpu", [TENSOR_SIZE], batching_enabled=True - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_tf_parameters/test.sh b/qa/L0_tf_parameters/test.sh deleted file mode 100755 index 133b6ef68d..0000000000 --- a/qa/L0_tf_parameters/test.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi -source ../common/util.sh - -export CUDA_VISIBLE_DEVICES=0 - -DATADIR=/data/inferenceserver/${REPO_VERSION}/qa_tf_parameters_repository -TEST_RESULT_FILE='test_results.txt' -CLIENT_LOG="./client.log" -TEST=tf_parameter_test.py -EXPECTED_NUM_TESTS="1" -MODEL_REPOSITORY=`pwd`/models -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_LOG="./inference_server.log" - -RET=0 - -rm -rf $SERVER_LOG $CLIENT_LOG models/ -cp -r $DATADIR models -SERVER_ARGS="--model-repository=$MODEL_REPOSITORY" -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python $TEST TFParameterTest.test_tf_variable_error>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -else - check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# Add the initialization operation -echo "{\"init_ops\": [\"init\"]}" > models/graphdef_variable/init_ops.json -echo "parameters: { key: \"TF_INIT_OPS_FILE\" value: { string_value:\"init_ops.json\" }}" >> models/graphdef_variable/config.pbtxt - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python $TEST TFParameterTest.test_tf_variable>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -else - check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# Move the initialization op to the model version folder. -mv models/graphdef_variable/init_ops.json models/graphdef_variable/1/ - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -set +e -python $TEST TFParameterTest.test_tf_variable>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -else - check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - cat $CLIENT_LOG - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET diff --git a/qa/L0_tf_parameters/tf_parameter_test.py b/qa/L0_tf_parameters/tf_parameter_test.py deleted file mode 100755 index f1a4621d93..0000000000 --- a/qa/L0_tf_parameters/tf_parameter_test.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import numpy as np -import test_util as tu -import tritonclient.http as tritonhttpclient -import tritonclient.utils - - -class TFParameterTest(tu.TestResultCollector): - def setUp(self): - self._client = tritonhttpclient.InferenceServerClient( - "localhost:8000", verbose=True - ) - - def _infer_helper(self): - # The model has a single variable which is added to the input. Since the - # variable is initialized to zero the input and output must match. - model_name = "graphdef_variable" - input = np.array([10], dtype=np.int32) - - inputs = [] - inputs.append(tritonhttpclient.InferInput("INPUT", input.shape, "INT32")) - inputs[-1].set_data_from_numpy(input) - - outputs = [] - outputs.append(tritonhttpclient.InferRequestedOutput("OUTPUT")) - - results = self._client.infer( - model_name=model_name, inputs=inputs, outputs=outputs - ) - output = results.as_numpy("OUTPUT") - np.testing.assert_array_equal(output, input) - - def test_tf_variable(self): - self._infer_helper() - - def test_tf_variable_error(self): - with self.assertRaises(tritonclient.utils.InferenceServerException) as e: - self._infer_helper() - self.assertIn( - "FAILED_PRECONDITION: Could not find variable VARIABLE. This " - + "could mean that the variable has been deleted. In TF1, it can " - + "also mean the variable is uninitialized.", - e.exception.message(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_tf_tag_sigdef/test.sh b/qa/L0_tf_tag_sigdef/test.sh deleted file mode 100755 index 32248c74ad..0000000000 --- a/qa/L0_tf_tag_sigdef/test.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# Copyright 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -TEST_RESULT_FILE='test_results.txt' -CLIENT_LOG="./client.log" -TEST=tf_tag_sigdef_test.py - -DATADIR=/data/inferenceserver/${REPO_VERSION}/qa_tf_tag_sigdef_repository -MODELDIR=`pwd`/models - -rm -rf $SERVER_LOG $CLIENT_LOG $MODELDIR -mkdir $MODELDIR -cp -r $DATADIR/* $MODELDIR - -EXPECTED_NUM_TESTS="4" -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=$MODELDIR --exit-timeout-secs=120" -SERVER_LOG="./inference_server.log" -source ../common/util.sh - -RET=0 - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - if [ `grep -c "configuration expects 2 inputs, model provides 1" $SERVER_LOG` != "0" ]; then - echo -e "*** FAILED: sig_tag_different_io config autocompleted with wrong model tag variant, failed to load.\n" - RET=1 - fi - cat $SERVER_LOG - exit 1 -fi - -set +e -python $TEST>$CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Failed\n***" - RET=1 -else - check_test_results $TEST_RESULT_FILE $EXPECTED_NUM_TESTS - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi -set -e - -kill $SERVER_PID -wait $SERVER_PID - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - cat $CLIENT_LOG - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET diff --git a/qa/L0_tf_tag_sigdef/tf_tag_sigdef_test.py b/qa/L0_tf_tag_sigdef/tf_tag_sigdef_test.py deleted file mode 100755 index b4a11ac04e..0000000000 --- a/qa/L0_tf_tag_sigdef/tf_tag_sigdef_test.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import numpy as np -import test_util as tu -import tritonhttpclient as httpclient - - -class TagSigdefTest(tu.TestResultCollector): - base_model_name = "sig_tag" - base_tag = "serve" - test_tag = "testTag" - base_sig_def = "serving_default" - test_sig_def = "testSigDef" - dims = 16 - - def _test_helper(self, modelVersion, tag, sig_def): - shape = [self.dims] - model_name = self.base_model_name + str(modelVersion) - # The multiplier is defined during model creation. See server/qa/common/gen_tag_sigdef.py - # for details - multiplier = modelVersion + 1 - output_name = "OUTPUT" - triton_client = httpclient.InferenceServerClient("localhost:8000", verbose=True) - inputs = [] - outputs = [] - inputs.append(httpclient.InferInput("INPUT", shape, "FP32")) - input_data = np.ones(shape=shape).astype(np.float32) - inputs[0].set_data_from_numpy(input_data, binary_data=True) - - outputs.append(httpclient.InferRequestedOutput(output_name, binary_data=True)) - results = triton_client.infer(model_name, inputs, outputs=outputs) - output_data = results.as_numpy(output_name) - test_output = input_data * multiplier - self.assertTrue(np.isclose(output_data, test_output).all()) - - def test_default(self): - self._test_helper(0, self.base_tag, self.base_sig_def) - - def test_sig_def(self): - self._test_helper(1, self.base_tag, self.test_sig_def) - - def test_tag(self): - self._test_helper(2, self.test_tag, self.base_sig_def) - - def test_tag_sig_def(self): - self._test_helper(3, self.test_tag, self.test_sig_def) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_tf_unknown_rank/test.sh b/qa/L0_tf_unknown_rank/test.sh deleted file mode 100755 index e279a46267..0000000000 --- a/qa/L0_tf_unknown_rank/test.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -TEST_RESULT_FILE='test_results.txt' -export CUDA_VISIBLE_DEVICES=0 - -DATADIR=/data/inferenceserver/${REPO_VERSION} - -CLIENT_LOG="./client.log" -UNKNOWN_RANK_TEST=tf_unknown_rank_test.py - -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models" -SERVER_LOG="./inference_server.log" -source ../common/util.sh - -rm -f ./*.log -rm -fr models && mkdir -p models -cp -r $DATADIR/tf_model_store2/unknown_rank_* models/ - -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -RET=0 - -set +e -python $UNKNOWN_RANK_TEST UnknownRankTest.test_success >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - cat $CLIENT_LOG - RET=1 -else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi - -python $UNKNOWN_RANK_TEST UnknownRankTest.test_wrong_input >> $CLIENT_LOG 2>&1 -if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - cat $CLIENT_LOG - RET=1 -else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi -fi - -set -e - -kill $SERVER_PID -wait $SERVER_PID - -# Try to load model with scalar tensor. The server should fail to load the model. -rm -rf scalar_repo; mkdir scalar_repo -cp -r $DATADIR/tf_model_store3/scalar_model scalar_repo/ -SERVER_ARGS="--model-repository=`pwd`/scalar_repo --strict-model-config=false" -run_server -if [ "$SERVER_PID" != "0" ]; then - echo -e "*** FAILED: unexpected success starting $SERVER" >> $CLIENT_LOG - RET=1 - kill $SERVER_PID - wait $SERVER_PID -else - ERROR_MESSAGE="Unable to autofill for 'scalar_model': the rank of model tensor 'x' is 0 and dimensions are not defined" - if [[ $(cat $SERVER_LOG | grep "${ERROR_MESSAGE}" | wc -l) -ne 2 ]]; then - echo -e "\n***\n*** Test Failed: "${ERROR_MESSAGE}" not found\n***" - cat $SERVER_LOG - RET=1 - fi -fi - - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET diff --git a/qa/L0_tf_unknown_rank/tf_unknown_rank_test.py b/qa/L0_tf_unknown_rank/tf_unknown_rank_test.py deleted file mode 100755 index add6b32c13..0000000000 --- a/qa/L0_tf_unknown_rank/tf_unknown_rank_test.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import numpy as np -import test_util as tu -import tritonhttpclient -from tritonclientutils import * - - -class UnknownRankTest(tu.TestResultCollector): - # helper function to generate requests to the server - def infer_unknown(self, model_name, tensor_shape): - print("About to run the test") - input_data = np.random.random_sample(tensor_shape).astype(np.float32) - client = tritonhttpclient.InferenceServerClient("localhost:8000") - inputs = [ - tritonhttpclient.InferInput( - "INPUT", input_data.shape, np_to_triton_dtype(input_data.dtype) - ) - ] - inputs[0].set_data_from_numpy(input_data) - results = client.infer(model_name, inputs) - self.assertTrue(np.array_equal(results.as_numpy("OUTPUT"), input_data)) - - def test_success(self): - model_name = "unknown_rank_success" - tensor_shape = 1 - try: - self.infer_unknown(model_name, tensor_shape) - except InferenceServerException as ex: - self.assertTrue(False, "unexpected error {}".format(ex)) - - def test_wrong_input(self): - model_name = "unknown_rank_wrong_output" - tensor_shape = (1, 2) - try: - self.infer_unknown(model_name, tensor_shape) - self.fail( - "Found success when expected failure with model given " - "wrong input tensor [1,2] for input [-1,1]." - ) - except InferenceServerException as ex: - self.assertIn( - "unexpected shape for input 'INPUT' for model " - "'unknown_rank_wrong_output'. Expected [1], got [1,2]", - ex.message(), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_tftrt_optimization/test.sh b/qa/L0_tftrt_optimization/test.sh deleted file mode 100755 index 04dcdc2f65..0000000000 --- a/qa/L0_tftrt_optimization/test.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION} -if [ "$#" -ge 1 ]; then - REPO_VERSION=$1 -fi -if [ -z "$REPO_VERSION" ]; then - echo -e "Repository version must be specified" - echo -e "\n***\n*** Test Failed\n***" - exit 1 -fi -if [ ! -z "$TEST_REPO_ARCH" ]; then - REPO_VERSION=${REPO_VERSION}_${TEST_REPO_ARCH} -fi - -export CUDA_VISIBLE_DEVICES=0 - -TEST_RESULT_FILE='test_results.txt' -DATADIR=/data/inferenceserver/${REPO_VERSION} - -CLIENT_LOG="./client.log" -TFTRT_OPTIMIZATION_TEST=tftrt_optimization_test.py - -SERVER=/opt/tritonserver/bin/tritonserver -SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --exit-on-error=false" -SERVER_LOG="./inference_server.log" -source ../common/util.sh - -RET=0 - -for MODEL in \ - graphdef_float32_float32_float32 \ - savedmodel_float32_float32_float32; do - rm -f ./*.log - rm -fr models && mkdir -p models - cp -r $DATADIR/qa_model_repository/${MODEL} \ - models/${MODEL}_def && \ - rm -fr models/${MODEL}_def/2 && \ - rm -fr models/${MODEL}_def/3 && \ - (cd models/${MODEL}_def && \ - sed -i 's/_float32_float32_float32/&_def/' config.pbtxt) && \ - # GPU execution accelerators with default setting - cp -r models/${MODEL}_def models/${MODEL}_trt && \ - (cd models/${MODEL}_trt && \ - sed -i 's/_float32_def/_float32_trt/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\"} ] } }" >> config.pbtxt) && \ - # GPU execution accelerators with correct parameters - cp -r models/${MODEL}_def models/${MODEL}_param && \ - (cd models/${MODEL}_param && \ - sed -i 's/_float32_def/_float32_param/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\" \ - parameters { key: \"precision_mode\" value: \"FP16\" } \ - parameters { key: \"minimum_segment_size\" value: \"1\" } }]}}" \ - >> config.pbtxt) && \ - # GPU execution accelerators with unknown parameters - cp -r models/${MODEL}_def models/${MODEL}_unknown_param && \ - (cd models/${MODEL}_unknown_param && \ - sed -i 's/_float32_def/_float32_unknown_param/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\" \ - parameters { key: \"precision_mode\" value: \"FP16\" } \ - parameters { key: \"segment_size\" value: \"1\" } }]}}" \ - >> config.pbtxt) && \ - # GPU execution accelerators with invalid parameters - cp -r models/${MODEL}_def models/${MODEL}_invalid_param && \ - (cd models/${MODEL}_invalid_param && \ - sed -i 's/_float32_def/_float32_invalid_param/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\" \ - parameters { key: \"precision_mode\" value: \"FP16\" } \ - parameters { key: \"max_workspace_size_bytes\" value: \"abc\" } }]}}" \ - >> config.pbtxt) && \ - # GPU execution accelerators on CPU context - cp -r models/${MODEL}_trt models/${MODEL}_cpu_trt && \ - (cd models/${MODEL}_cpu_trt && \ - sed -i 's/_float32_trt/_float32_cpu_trt/' \ - config.pbtxt && \ - echo "instance_group [ { kind: KIND_CPU }]" >> config.pbtxt) && \ - # CPU execution accelerators - cp -r models/${MODEL}_def models/${MODEL}_openvino && \ - (cd models/${MODEL}_openvino && \ - sed -i 's/_float32_def/_float32_openvino/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { cpu_execution_accelerator : [ { name : \"openvino\" } ] } }" >> config.pbtxt) && \ - # Unknown GPU execution accelerator - cp -r models/${MODEL}_def models/${MODEL}_unknown_gpu && \ - (cd models/${MODEL}_unknown_gpu && \ - sed -i 's/_float32_def/_float32_unknown_gpu/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"unknown_gpu\" } ] } }" >> config.pbtxt) && \ - # Unknown CPU execution accelerators - cp -r models/${MODEL}_def models/${MODEL}_unknown_cpu && \ - (cd models/${MODEL}_unknown_cpu && \ - sed -i 's/_float32_def/_float32_unknown_cpu/' \ - config.pbtxt && \ - echo "optimization { execution_accelerators { cpu_execution_accelerator : [ { name : \"unknown_cpu\" } ] } }" >> config.pbtxt) - - run_server_tolive - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - - grep "TensorRT Execution Accelerator is set for ${MODEL}_trt" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected TensorRT Execution Accelerator is set\n***" - RET=1 - fi - - grep "TensorRT Execution Accelerator is set for ${MODEL}_param" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected TensorRT Execution Accelerator is set\n***" - RET=1 - fi - - grep "failed to load '${MODEL}_unknown_param' version 1: Invalid argument: unknown parameter 'segment_size' is provided for TensorRT Execution Accelerator" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected unknown parameter 'segment_size' returns error\n***" - RET=1 - fi - - grep "failed to load '${MODEL}_invalid_param' version 1: Invalid argument: failed to convert 'abc' to long long integral number" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected invalid parameter 'abc' returns error\n***" - RET=1 - fi - - grep "GPU Execution Accelerator will be ignored for model instance on CPU" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected logged warning: GPU Execution Accelerator will be ignored for model instance on CPU\n***" - RET=1 - fi - - grep "failed to load '${MODEL}_openvino' version 1: Invalid argument: CPU Execution Accelerator is not supported in TensorFlow backend" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected CPU Execution Accelerator returns error\n***" - RET=1 - fi - - grep "failed to load '${MODEL}_unknown_gpu' version 1: Invalid argument: unknown Execution Accelerator 'unknown_gpu' is requested" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected 'unknown_gpu' Execution Accelerator returns error\n***" - RET=1 - fi - grep "failed to load '${MODEL}_unknown_cpu' version 1: Invalid argument: CPU Execution Accelerator is not supported in TensorFlow backend" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected 'unknown_cpu' Execution Accelerator returns error\n***" - RET=1 - fi - - TEST_TYPE=test_graphdef && \ - [[ "$MODEL" == "savedmodel_float32_float32_float32" ]] && \ - TEST_TYPE=test_savedmodel - echo "Test: $MODEL" >>$CLIENT_LOG - python $TFTRT_OPTIMIZATION_TEST TFTRTOptimizationTest.$TEST_TYPE \ - >>$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - cat $CLIENT_LOG - RET=1 - else - check_test_results $TEST_RESULT_FILE 1 - if [ $? -ne 0 ]; then - cat $CLIENT_LOG - echo -e "\n***\n*** Test Result Verification Failed\n***" - RET=1 - fi - fi - - set -e - - kill $SERVER_PID - wait $SERVER_PID -done - -if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" -else - echo -e "\n***\n*** Test FAILED\n***" -fi - -exit $RET diff --git a/qa/L0_tftrt_optimization/tftrt_optimization_test.py b/qa/L0_tftrt_optimization/tftrt_optimization_test.py deleted file mode 100755 index 9e59677317..0000000000 --- a/qa/L0_tftrt_optimization/tftrt_optimization_test.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import sys - -sys.path.append("../common") - -import unittest - -import numpy as np -import test_util as tu -import tritonhttpclient as httpclient - - -class TFTRTOptimizationTest(tu.TestResultCollector): - def setUp(self): - self.input0_ = np.arange(start=0, stop=16, dtype=np.float32).reshape(1, 16) - self.input1_ = np.ones(shape=16, dtype=np.float32).reshape(1, 16) - self.expected_output0_ = self.input0_ + self.input1_ - self.expected_output1_ = self.input0_ - self.input1_ - - def _addsub_infer(self, model_name): - triton_client = httpclient.InferenceServerClient("localhost:8000", verbose=True) - - inputs = [] - outputs = [] - inputs.append(httpclient.InferInput("INPUT0", [1, 16], "FP32")) - inputs.append(httpclient.InferInput("INPUT1", [1, 16], "FP32")) - - # Initialize the data - inputs[0].set_data_from_numpy(self.input0_, binary_data=True) - inputs[1].set_data_from_numpy(self.input1_, binary_data=False) - - outputs.append(httpclient.InferRequestedOutput("OUTPUT0", binary_data=True)) - outputs.append(httpclient.InferRequestedOutput("OUTPUT1", binary_data=True)) - - results = triton_client.infer(model_name, inputs, outputs=outputs) - - output0_data = results.as_numpy("OUTPUT0") - output1_data = results.as_numpy("OUTPUT1") - - self.assertTrue( - np.array_equal(self.expected_output0_, output0_data), "incorrect sum" - ) - self.assertTrue( - np.array_equal(self.expected_output1_, output1_data), "incorrect difference" - ) - - def test_graphdef(self): - self._addsub_infer("graphdef_float32_float32_float32_trt") - self._addsub_infer("graphdef_float32_float32_float32_param") - - def test_savedmodel(self): - self._addsub_infer("savedmodel_float32_float32_float32_trt") - self._addsub_infer("savedmodel_float32_float32_float32_param") - - -if __name__ == "__main__": - unittest.main() diff --git a/qa/L0_warmup/test.sh b/qa/L0_warmup/test.sh index d307b7ce07..c3e6885062 100755 --- a/qa/L0_warmup/test.sh +++ b/qa/L0_warmup/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -51,7 +51,7 @@ IMAGE="../images/vulture.jpeg" DATADIR=`pwd`/models # If BACKENDS not specified, set to all -BACKENDS=${BACKENDS:="graphdef savedmodel onnx libtorch plan"} +BACKENDS=${BACKENDS:="onnx libtorch plan"} SERVER=/opt/tritonserver/bin/tritonserver SERVER_ARGS="--model-repository=$DATADIR --log-verbose=1 --exit-timeout-secs=120" @@ -177,7 +177,7 @@ for BACKEND in ${BACKENDS}; do # Test for variable-size data type (string) rm -fr models && mkdir models - SUPPORT_STRING=0 && ([[ $BACKEND == "savedmodel" ]] || [[ $BACKEND == "onnx" ]] || [[ $BACKEND == "savedmodel" ]]) && SUPPORT_STRING=1 + SUPPORT_STRING=0 && ([[ $BACKEND == "onnx" ]]) && SUPPORT_STRING=1 if [ "$SUPPORT_STRING" == "1" ] ; then cp -r /data/inferenceserver/${REPO_VERSION}/qa_sequence_model_repository/${BACKEND}_sequence_object models/. cp -r /data/inferenceserver/${REPO_VERSION}/qa_identity_model_repository/${BACKEND}_zero_1_object models/. @@ -287,80 +287,85 @@ for BACKEND in ${BACKENDS}; do wait $SERVER_PID fi - if [ "$BACKEND" == "graphdef" ]; then - # Show effect of warmup by using a TF model with TF-TRT optimization which is - # known to be slow on first inference. - # Note: model can be obatined via the fetching script in docs/example - rm -fr models && \ - mkdir models && \ - cp -r /data/inferenceserver/${REPO_VERSION}/tf_model_store/inception_v3_graphdef models/. - - # Enable TF-TRT optimization - (cd models/inception_v3_graphdef && \ - echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\"} ] } }" >> config.pbtxt) - - # Duplicate the same model with warmup enabled - cp -r models/inception_v3_graphdef models/inception_v3_warmup && - (cd models/inception_v3_warmup && \ - sed -i 's/inception_v3_graphdef/inception_v3_warmup/' config.pbtxt) - - (cd models/inception_v3_warmup && \ - echo 'model_warmup [{' >> config.pbtxt && \ - echo ' name : "image sample"' >> config.pbtxt && \ - echo ' batch_size: 1' >> config.pbtxt && \ - echo ' inputs {' >> config.pbtxt && \ - echo ' key: "input"' >> config.pbtxt && \ - echo ' value: {' >> config.pbtxt && \ - echo ' data_type: TYPE_FP32' >> config.pbtxt && \ - echo ' dims: [ 299, 299, 3 ]' >> config.pbtxt && \ - echo ' input_data_file: "raw_mug_data"' >> config.pbtxt && \ - echo ' }' >> config.pbtxt && \ - echo ' }' >> config.pbtxt && \ - echo '}]' >> config.pbtxt ) - - # prepare provided data instead of synthetic one - mkdir -p models/inception_v3_warmup/warmup && \ - cp raw_mug_data models/inception_v3_warmup/warmup/. - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - set +e - - grep "is running warmup sample 'image sample'" $SERVER_LOG - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Failed. Expected warmup for image model\n***" - RET=1 - fi - grep "failed to run warmup" $SERVER_LOG - if [ $? -eq 0 ]; then - echo -e "\n***\n*** Failed. Expected no warmup error\n***" - RET=1 - fi - - # Time the first inference for both models - time $CLIENT -m inception_v3_graphdef -s INCEPTION $IMAGE -i grpc -u localhost:8001 >>$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - cat $CLIENT_LOG - RET=1 - fi - time $CLIENT -m inception_v3_warmup -s INCEPTION $IMAGE -i grpc -u localhost:8001 >>$CLIENT_LOG 2>&1 - if [ $? -ne 0 ]; then - echo -e "\n***\n*** Test Failed\n***" - cat $CLIENT_LOG - RET=1 - fi - - set -e - - kill $SERVER_PID - wait $SERVER_PID - fi + # FIXME: This section of code doesn't check if the warmup model + # is faster than the fresh model. Thus we are not losing any coverage + # by commenting it out. The functionality of the warmup methods are + # covered by other parts of this test which will fail if the *functionality* + # breaks. + # if [ "$BACKEND" == "graphdef" ]; then + # # Show effect of warmup by using a TF model with TF-TRT optimization which is + # # known to be slow on first inference. + # # Note: model can be obatined via the fetching script in docs/example + # rm -fr models && \ + # mkdir models && \ + # cp -r /data/inferenceserver/${REPO_VERSION}/tf_model_store/inception_v3_graphdef models/. + + # # Enable TF-TRT optimization + # (cd models/inception_v3_graphdef && \ + # echo "optimization { execution_accelerators { gpu_execution_accelerator : [ { name : \"tensorrt\"} ] } }" >> config.pbtxt) + + # # Duplicate the same model with warmup enabled + # cp -r models/inception_v3_graphdef models/inception_v3_warmup && + # (cd models/inception_v3_warmup && \ + # sed -i 's/inception_v3_graphdef/inception_v3_warmup/' config.pbtxt) + + # (cd models/inception_v3_warmup && \ + # echo 'model_warmup [{' >> config.pbtxt && \ + # echo ' name : "image sample"' >> config.pbtxt && \ + # echo ' batch_size: 1' >> config.pbtxt && \ + # echo ' inputs {' >> config.pbtxt && \ + # echo ' key: "input"' >> config.pbtxt && \ + # echo ' value: {' >> config.pbtxt && \ + # echo ' data_type: TYPE_FP32' >> config.pbtxt && \ + # echo ' dims: [ 299, 299, 3 ]' >> config.pbtxt && \ + # echo ' input_data_file: "raw_mug_data"' >> config.pbtxt && \ + # echo ' }' >> config.pbtxt && \ + # echo ' }' >> config.pbtxt && \ + # echo '}]' >> config.pbtxt ) + + # # prepare provided data instead of synthetic one + # mkdir -p models/inception_v3_warmup/warmup && \ + # cp raw_mug_data models/inception_v3_warmup/warmup/. + + # run_server + # if [ "$SERVER_PID" == "0" ]; then + # echo -e "\n***\n*** Failed to start $SERVER\n***" + # cat $SERVER_LOG + # exit 1 + # fi + + # set +e + + # grep "is running warmup sample 'image sample'" $SERVER_LOG + # if [ $? -ne 0 ]; then + # echo -e "\n***\n*** Failed. Expected warmup for image model\n***" + # RET=1 + # fi + # grep "failed to run warmup" $SERVER_LOG + # if [ $? -eq 0 ]; then + # echo -e "\n***\n*** Failed. Expected no warmup error\n***" + # RET=1 + # fi + + # # Time the first inference for both models + # time $CLIENT -m inception_v3_graphdef -s INCEPTION $IMAGE -i grpc -u localhost:8001 >>$CLIENT_LOG 2>&1 + # if [ $? -ne 0 ]; then + # echo -e "\n***\n*** Test Failed\n***" + # cat $CLIENT_LOG + # RET=1 + # fi + # time $CLIENT -m inception_v3_warmup -s INCEPTION $IMAGE -i grpc -u localhost:8001 >>$CLIENT_LOG 2>&1 + # if [ $? -ne 0 ]; then + # echo -e "\n***\n*** Test Failed\n***" + # cat $CLIENT_LOG + # RET=1 + # fi + + # set -e + + # kill $SERVER_PID + # wait $SERVER_PID + # fi done # Test warmup sample failure diff --git a/qa/common/busy_op_kernel.cc b/qa/common/busy_op_kernel.cc deleted file mode 100644 index 119ed0a1ce..0000000000 --- a/qa/common/busy_op_kernel.cc +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// * Neither the name of NVIDIA CORPORATION nor the names of its -// contributors may be used to endorse or promote products derived -// from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include - -#include "tensorflow/core/framework/device_base.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_shape.h" - -using namespace tensorflow; // NOLINT(build/namespaces) - -REGISTER_OP("BusyLoop").Input("input: int32").Output("output: int32").Doc(R"doc( -Busy waits for input number of clock cycles -)doc"); - -void BusyLoopKernelLauncher( - const Eigen::GpuDevice& device, const int* num_delay_cycles, int* out); - -class BusyLoopOp : public OpKernel { - public: - explicit BusyLoopOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override - { - // Grab the input - const Tensor& input_tensor = context->input(0); - auto num_delay_cycles = input_tensor.flat(); - - // Create dummy output - Tensor* output_tensor = nullptr; - OP_REQUIRES_OK( - context, - context->allocate_output(0, input_tensor.shape(), &output_tensor)); - auto output = output_tensor->template flat(); - - // Verify input dimension - OP_REQUIRES( - context, TensorShapeUtils::IsVector(input_tensor.shape()), - errors::InvalidArgument( - "BusyLoop expects a single value as a 1-D Vector")); - - // Call the cuda kernel launcher - BusyLoopKernelLauncher( - context->eigen_device(), num_delay_cycles.data(), - output.data()); - } -}; - -REGISTER_KERNEL_BUILDER(Name("BusyLoop").Device(DEVICE_GPU), BusyLoopOp); diff --git a/qa/common/check_copyright.py b/qa/common/check_copyright.py index 95694dc460..99606a1c81 100755 --- a/qa/common/check_copyright.py +++ b/qa/common/check_copyright.py @@ -63,7 +63,6 @@ "docs/repositories.txt", "docs/exclusions.txt", "docker", - "qa/common/cuda_op_kernel.cu.cc.patch", "qa/ensemble_models/mix_platform_float32_float32_float32/output0_labels.txt", "qa/ensemble_models/mix_type_int32_float32_float32/output0_labels.txt", "qa/ensemble_models/mix_ensemble_int32_float32_float32/output0_labels.txt", diff --git a/qa/common/check_valgrind_log.py b/qa/common/check_valgrind_log.py index 201d0e922c..84eecd4989 100755 --- a/qa/common/check_valgrind_log.py +++ b/qa/common/check_valgrind_log.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,19 +31,12 @@ # Check the valgrind logs for memory leaks, ignoring known memory leaks # * cnmem https://github.com/NVIDIA/cnmem/issues/12 -# * Tensorflow::NewSession # * dl-open leak could be due to https://bugs.kde.org/show_bug.cgi?id=358980 -# * dlerror leak in tensorflow::HadoopFileSystem::HadoopFileSystem() -# -> tensorflow::LibHDFS::LoadAndBind()::{lambda(char const*, void**)#1}::operator()(char const*, void**) -# -> tensorflow::internal::LoadLibrary -# -> dlerror LEAK_WHITE_LIST = [ "cnmem", - "tensorflow::NewSession", "dl-init", "dl-open", - "dlerror", "libtorch", ] diff --git a/qa/common/cuda_op_kernel.cu.cc.patch b/qa/common/cuda_op_kernel.cu.cc.patch deleted file mode 100644 index e28072670d..0000000000 --- a/qa/common/cuda_op_kernel.cu.cc.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc b/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc -index a9d66f9..a92e218 100644 ---- a/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc -+++ b/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc -@@ -14,10 +14,12 @@ limitations under the License. - ==============================================================================*/ - - #if GOOGLE_CUDA --#define EIGEN_USE_GPU --#include "unsupported/Eigen/CXX11/Tensor" // from @eigen_archive --#include "tensorflow/core/util/gpu_kernel_helper.h" --#include "tensorflow/core/util/gpu_launch_config.h" -+//#define EIGEN_USE_GPU -+//#include "unsupported/Eigen/CXX11/Tensor" -+//#include "tensorflow/core/util/gpu_kernel_helper.h" -+//#include "tensorflow/core/util/gpu_launch_config.h" -+#include -+#include - - __global__ void AddOneKernel(const int* in, const int N, int* out) { - for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; -@@ -27,8 +29,9 @@ __global__ void AddOneKernel(const int* in, const int N, int* out) { - } - - void AddOneKernelLauncher(const int* in, const int N, int* out) { -- TF_CHECK_OK(::tensorflow::GpuLaunchKernel(AddOneKernel, 32, 256, 0, nullptr, -- in, N, out)); -+ int block_size = std::min(N, 1024); -+ int grid_size = (N + block_size - 1) / block_size; -+ AddOneKernel<<>>(in, N, out); - } - - #endif diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index 5b7709504b..d53702d604 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -115,34 +115,6 @@ def np_to_trt_dtype(np_dtype): return None -def np_to_tf_dtype(np_dtype): - import tensorflow as tf - - if np_dtype == bool: - return tf.bool - elif np_dtype == np.int8: - return tf.int8 - elif np_dtype == np.int16: - return tf.int16 - elif np_dtype == np.int32: - return tf.int32 - elif np_dtype == np.int64: - return tf.int64 - elif np_dtype == np.uint8: - return tf.uint8 - elif np_dtype == np.uint16: - return tf.uint16 - elif np_dtype == np.float16: - return tf.float16 - elif np_dtype == np.float32: - return tf.float32 - elif np_dtype == np.float64: - return tf.float64 - elif np_dtype == np_dtype_string: - return tf.string - return None - - def np_to_torch_dtype(np_dtype): import torch diff --git a/qa/common/gen_ensemble_model_utils.py b/qa/common/gen_ensemble_model_utils.py index dd4f6e326a..fec8f0bf92 100755 --- a/qa/common/gen_ensemble_model_utils.py +++ b/qa/common/gen_ensemble_model_utils.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -43,8 +43,6 @@ def fixed_to_variable_size(shape): def platform_types_and_validation(): res = [ - ("graphdef", tu.validate_for_tf_model), - ("savedmodel", tu.validate_for_tf_model), ("plan", tu.validate_for_trt_model), ("onnx", tu.validate_for_onnx_model), ("libtorch", tu.validate_for_libtorch_model), diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 47fd8c758c..1dda5b72cf 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.02} +TRITON_VERSION=${TRITON_VERSION:=25.03} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image @@ -108,7 +108,7 @@ cat > $TRT_MODEL_SCRIPT < $TFSCRIPT <4.24.0" - -TF_CFLAGS=\$(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_compile_flags()))') -TF_LFLAGS=\$(python -c 'import tensorflow as tf; print(" ".join(tf.sysconfig.get_link_flags()))') - -# No CUDA -cd /tmp -cp /opt/tensorflow/tensorflow-source/tensorflow/examples/adding_an_op/zero_out_op_kernel_1.cc . -g++ -std=${STD_FLAG} -O2 -shared -fPIC zero_out_op_kernel_1.cc -o \$DESTDIR/libzeroout.so \${TF_CFLAGS[@]} \${TF_LFLAGS[@]} - -# CUDA. Need to patch so that we can build it outside of bazel/TF -cp /opt/tensorflow/tensorflow-source/tensorflow/examples/adding_an_op/cuda_op_kernel.cc . -cp /opt/tensorflow/tensorflow-source/tensorflow/examples/adding_an_op/cuda_op_kernel.cu.cc . -patch -i $VOLUME_SRCDIR/cuda_op_kernel.cu.cc.patch cuda_op_kernel.cu.cc -nvcc --expt-relaxed-constexpr -std=${STD_FLAG} -O2 -c -arch=all -o cuda_op_kernel.cu.o cuda_op_kernel.cu.cc \${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC -g++ -std=${STD_FLAG} -shared -o \$DESTDIR/libcudaop.so cuda_op_kernel.cc cuda_op_kernel.cu.o \${TF_CFLAGS[@]} -fPIC -L/usr/local/cuda/lib64 -lcudart \${TF_LFLAGS[@]} - -cp $VOLUME_SRCDIR/busy_op_kernel.cc . -cp $VOLUME_SRCDIR/busy_op_kernel.cu.cc . -nvcc --expt-relaxed-constexpr -std=${STD_FLAG} -O2 -c -arch=all -o busy_op_kernel.cu.o busy_op_kernel.cu.cc \${TF_CFLAGS[@]} -D GOOGLE_CUDA=1 -x cu -Xcompiler -fPIC -g++ -std=${STD_FLAG} -shared -o \$DESTDIR/libbusyop.so busy_op_kernel.cc busy_op_kernel.cu.o \${TF_CFLAGS[@]} -fPIC -L/usr/local/cuda/lib64 -lcudart \${TF_LFLAGS[@]} - -python3 $VOLUME_SRCDIR/gen_qa_custom_ops_models.py --graphdef --savedmodel \ - --models_dir=\$DESTDIR --zero_out_lib_path=\$DESTDIR/libzeroout.so \ - --cuda_op_lib_path=\$DESTDIR/libcudaop.so \ - --busy_op_lib_path=\$DESTDIR/libbusyop.so -chmod -R 777 \$DESTDIR -EOF - -chmod a+x $TFSCRIPT -if [ $? -ne 0 ]; then - echo -e "Failed: chmod" - exit 1 -fi - -docker cp $TFSCRIPT $DOCKER_VOLUME_CONTAINER:$VOLUME_SRCDIR/$TFSCRIPT - -docker pull $TENSORFLOW_IMAGE - -echo -e "\033[34m[ INFO ] - Running: $TFSCRIPT \033[0m " - -docker run \ - --rm \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - -e DESTDIR=$VOLUME_DESTDIR/tf_custom_ops \ - $TENSORFLOW_IMAGE \ - bash -xe $VOLUME_SRCDIR/$TFSCRIPT - -if [ $? -ne 0 ]; then - echo -e "Failed" - exit 1 -fi - # PyTorch cat > $PYTSCRIPT < 0 - else "" - ), - "fp32" if dtype == np.float32 else "int32", - "fp32" if dtype == np.float32 else "int32", - "fp32" if dtype == np.float32 else "int32", - np_to_model_dtype(dtype), - tu.shape_to_dims_str(shape), - np_to_model_dtype(dtype), - ) - - try: - os.makedirs(config_dir) - except OSError as ex: - pass # ignore existing dir - - with open(config_dir + "/config.pbtxt", "w") as cfile: - cfile.write(config) - - def create_plan_shape_tensor_modelfile( models_dir, model_version, max_batch, dtype, shape, shape_tensor_input_dtype ): @@ -1433,20 +1161,6 @@ def create_shape_tensor_models( def create_models(models_dir, dtype, shape, no_batch=True): model_version = 1 - if FLAGS.graphdef: - create_tf_modelconfig(False, models_dir, model_version, 8, dtype, shape) - create_tf_modelfile(False, models_dir, model_version, 8, dtype, shape) - if no_batch: - create_tf_modelconfig(False, models_dir, model_version, 0, dtype, shape) - create_tf_modelfile(False, models_dir, model_version, 0, dtype, shape) - - if FLAGS.savedmodel: - create_tf_modelconfig(True, models_dir, model_version, 8, dtype, shape) - create_tf_modelfile(True, models_dir, model_version, 8, dtype, shape) - if no_batch: - create_tf_modelconfig(True, models_dir, model_version, 0, dtype, shape) - create_tf_modelfile(True, models_dir, model_version, 0, dtype, shape) - if FLAGS.tensorrt: suffix = [] if dtype == np.int8: @@ -1485,18 +1199,6 @@ def create_models(models_dir, dtype, shape, no_batch=True): parser.add_argument( "--models_dir", type=str, required=True, help="Top-level model directory" ) - parser.add_argument( - "--graphdef", - required=False, - action="store_true", - help="Generate GraphDef models", - ) - parser.add_argument( - "--savedmodel", - required=False, - action="store_true", - help="Generate SavedModel models", - ) parser.add_argument( "--tensorrt", required=False, @@ -1539,11 +1241,6 @@ def create_models(models_dir, dtype, shape, no_batch=True): ) FLAGS, unparsed = parser.parse_known_args() - if FLAGS.graphdef or FLAGS.savedmodel: - import tensorflow as tf - from tensorflow.python.framework import graph_io - - tf.compat.v1.disable_eager_execution() if FLAGS.tensorrt or FLAGS.tensorrt_shape_io: import tensorrt as trt if FLAGS.onnx: diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index e74a4f62b7..5fa7b7ab01 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -35,7 +35,6 @@ from gen_common import ( np_to_model_dtype, np_to_onnx_dtype, - np_to_tf_dtype, np_to_trt_dtype, openvino_save_model, ) @@ -45,138 +44,6 @@ from typing import List, Tuple -def create_tf_modelfile( - create_savedmodel, models_dir, model_version, io_cnt, max_batch, dtype, shape -): - if not tu.validate_for_tf_model(dtype, dtype, dtype, shape, shape, shape): - return - - tf_dtype = np_to_tf_dtype(dtype) - - # Create the model that copies inputs to corresponding outputs. - tf.compat.v1.reset_default_graph() - for io_num in range(io_cnt): - input_name = "INPUT{}".format(io_num) - output_name = "OUTPUT{}".format(io_num) - if max_batch == 0: - tin = tf.compat.v1.placeholder( - tf_dtype, tu.shape_to_tf_shape(shape), input_name - ) - else: - tin = tf.compat.v1.placeholder( - tf_dtype, - [ - None, - ] - + tu.shape_to_tf_shape(shape), - input_name, - ) - toutput = tf.identity(tin, name=output_name) - - # Use model name based on io_cnt and non-batching variant - if create_savedmodel: - model_name = tu.get_zero_model_name( - "savedmodel_nobatch" if max_batch == 0 else "savedmodel", io_cnt, dtype - ) - else: - model_name = tu.get_zero_model_name( - "graphdef_nobatch" if max_batch == 0 else "graphdef", io_cnt, dtype - ) - - model_version_dir = os.path.join(models_dir, model_name, str(model_version)) - os.makedirs(model_version_dir, exist_ok=True) - - if create_savedmodel: - with tf.compat.v1.Session() as sess: - input_dict = {} - output_dict = {} - for io_num in range(io_cnt): - input_name = "INPUT{}".format(io_num) - output_name = "OUTPUT{}".format(io_num) - input_tensor = tf.compat.v1.get_default_graph().get_tensor_by_name( - input_name + ":0" - ) - output_tensor = tf.compat.v1.get_default_graph().get_tensor_by_name( - output_name + ":0" - ) - input_dict[input_name] = input_tensor - output_dict[output_name] = output_tensor - tf.compat.v1.saved_model.simple_save( - sess, - model_version_dir + "/model.savedmodel", - inputs=input_dict, - outputs=output_dict, - ) - else: - with tf.compat.v1.Session() as sess: - graph_io.write_graph( - sess.graph.as_graph_def(), - model_version_dir, - "model.graphdef", - as_text=False, - ) - - -def create_tf_modelconfig( - create_savedmodel, models_dir, model_version, io_cnt, max_batch, dtype, shape -): - if not tu.validate_for_tf_model(dtype, dtype, dtype, shape, shape, shape): - return - - shape_str = tu.shape_to_dims_str(shape) - - # Use a different model name for the non-batching variant - if create_savedmodel: - model_name = tu.get_zero_model_name( - "savedmodel_nobatch" if max_batch == 0 else "savedmodel", io_cnt, dtype - ) - else: - model_name = tu.get_zero_model_name( - "graphdef_nobatch" if max_batch == 0 else "graphdef", io_cnt, dtype - ) - - config_dir = os.path.join(models_dir, model_name) - config = """ -name: "{}" -platform: "{}" -max_batch_size: {} -""".format( - model_name, - "tensorflow_savedmodel" if create_savedmodel else "tensorflow_graphdef", - max_batch, - ) - - for io_num in range(io_cnt): - config += """ -input [ - {{ - name: "INPUT{}" - data_type: {} - dims: [ {} ] - }} -] -output [ - {{ - name: "OUTPUT{}" - data_type: {} - dims: [ {} ] - }} -] -""".format( - io_num, - np_to_model_dtype(dtype), - shape_str, - io_num, - np_to_model_dtype(dtype), - shape_str, - ) - - os.makedirs(config_dir, exist_ok=True) - - with open(config_dir + "/config.pbtxt", "w") as cfile: - cfile.write(config) - - def create_ensemble_modelfile( create_savedmodel, models_dir, model_version, io_cnt, max_batch, dtype, shape ): @@ -1107,28 +974,6 @@ def create_shape_tensor_models( def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): model_version = 1 - if FLAGS.graphdef: - create_tf_modelconfig(False, models_dir, model_version, io_cnt, 8, dtype, shape) - create_tf_modelfile(False, models_dir, model_version, io_cnt, 8, dtype, shape) - if no_batch: - create_tf_modelconfig( - False, models_dir, model_version, io_cnt, 0, dtype, shape - ) - create_tf_modelfile( - False, models_dir, model_version, io_cnt, 0, dtype, shape - ) - - if FLAGS.savedmodel: - create_tf_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) - create_tf_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) - if no_batch: - create_tf_modelconfig( - True, models_dir, model_version, io_cnt, 0, dtype, shape - ) - create_tf_modelfile( - True, models_dir, model_version, io_cnt, 0, dtype, shape - ) - if FLAGS.onnx: create_onnx_modelconfig( True, models_dir, model_version, io_cnt, 8, dtype, shape @@ -1222,23 +1067,13 @@ def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): ) +# FIXME: The function signatures require a `savedmodel` boolean flag +# on all of them even though Tensorflow has been deprecated since 25.03 if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--models_dir", type=str, required=True, help="Top-level model directory" ) - parser.add_argument( - "--graphdef", - required=False, - action="store_true", - help="Generate GraphDef models", - ) - parser.add_argument( - "--savedmodel", - required=False, - action="store_true", - help="Generate SavedModel models", - ) parser.add_argument( "--onnx", required=False, @@ -1296,11 +1131,6 @@ def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): ) FLAGS, unparsed = parser.parse_known_args() - if FLAGS.graphdef or FLAGS.savedmodel: - import tensorflow as tf - from tensorflow.python.framework import graph_io - - tf.compat.v1.disable_eager_execution() if FLAGS.onnx: import onnx if FLAGS.libtorch: diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index 793998bdea..c3429d6012 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2021-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -1276,18 +1276,6 @@ def create_models(models_dir, dtype, shape, initial_state, no_batch=True): parser.add_argument( "--models_dir", type=str, required=True, help="Top-level model directory" ) - parser.add_argument( - "--graphdef", - required=False, - action="store_true", - help="Generate GraphDef models", - ) - parser.add_argument( - "--savedmodel", - required=False, - action="store_true", - help="Generate SavedModel models", - ) parser.add_argument( "--tensorrt", required=False, diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 38bebb7f43..0b5e0e9e1d 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -28,9 +28,9 @@ ############################################################################ ## This script generates the model repository needed by some of the ## tritonserver CI tests. Generating these models requires using -## the TensorFlow and PyTorch containers. +## the PyTorch container. ## -## 1. Update TENSORRT_IMAGE, PYTORCH_IMAGE and TENSORFLOW_IMAGE to +## 1. Update TENSORRT_IMAGE and PYTORCH_IMAGE to ## match what is being used by the tritonserver release being ## tested. ## @@ -48,7 +48,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.02} +TRITON_VERSION=${TRITON_VERSION:=25.03} # ONNX. Use ONNX_OPSET 0 to use the default for ONNX version ONNX_VERSION=1.16.1 @@ -59,7 +59,6 @@ OPENVINO_VERSION=2024.5.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} -TENSORFLOW_IMAGE=${TENSORFLOW_IMAGE:=nvcr.io/nvidia/tensorflow:$TRITON_VERSION-tf2-py3} TENSORRT_IMAGE=${TENSORRT_IMAGE:=nvcr.io/nvidia/tensorrt:$TRITON_VERSION-py3} CUDA_DEVICE=${NV_GPU:=0} @@ -109,9 +108,7 @@ VOLUME_SRCDIR=${VOLUME_SRCDIR:=$VOLUME_BUILD_DIR/gen_srcdir} VOLUME_DESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_model_repository VOLUME_VARDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_variable_model_repository VOLUME_IDENTITYDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_identity_model_repository -VOLUME_SIGDEFDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_tf_tag_sigdef_repository VOLUME_IDENTITYBIGDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_identity_big_model_repository -VOLUME_TFPARAMETERSDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_tf_parameters_repository VOLUME_SHAPEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_shapetensor_model_repository VOLUME_RESHAPEDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_reshape_model_repository VOLUME_SEQDESTDIR=$VOLUME_BUILD_DIR/$TRITON_VERSION/qa_sequence_model_repository @@ -172,7 +169,6 @@ docker run \ ONNXSCRIPT=gen.ONNXRuntime.gen_qa_model_repository.cmds OPENVINOSCRIPT=gen.OpenVINO.gen_qa_model_repository.cmds TORCHSCRIPT=gen.PyTorch.gen_qa_model_repository.cmds -TFSCRIPT=gen.TensorFlow.gen_qa_model_repository.cmds TRTSCRIPT=gen.TensorRT.gen_qa_model_repository.cmds # OPENVINO @@ -184,7 +180,7 @@ cat > $OPENVINOSCRIPT < $ONNXSCRIPT < $TORCHSCRIPT < $TFSCRIPT <4.24.0" - -python3 $VOLUME_SRCDIR/gen_qa_models.py --graphdef --savedmodel --models_dir=$VOLUME_DESTDIR -chmod -R 777 $VOLUME_DESTDIR -python3 $VOLUME_SRCDIR/gen_qa_models.py --graphdef --savedmodel --variable --models_dir=$VOLUME_VARDESTDIR -chmod -R 777 $VOLUME_VARDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --graphdef --savedmodel --models_dir=$VOLUME_IDENTITYDESTDIR -chmod -R 777 $VOLUME_IDENTITYDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_reshape_models.py --graphdef --savedmodel --variable --models_dir=$VOLUME_RESHAPEDESTDIR -chmod -R 777 $VOLUME_RESHAPEDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --models_dir=$VOLUME_SEQDESTDIR -chmod -R 777 $VOLUME_SEQDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --graphdef --savedmodel --variable --models_dir=$VOLUME_VARSEQDESTDIR -chmod -R 777 $VOLUME_VARSEQDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_dyna_sequence_models.py --graphdef --savedmodel --models_dir=$VOLUME_DYNASEQDESTDIR -chmod -R 777 $VOLUME_DYNASEQDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_noshape_models.py --savedmodel --models_dir=$VOLUME_NOSHAPEDESTDIR -chmod -R 777 $VOLUME_NOSHAPEDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_ragged_models.py --savedmodel --models_dir=$VOLUME_RAGGEDDESTDIR -chmod -R 777 $VOLUME_RAGGEDDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_models.py --ensemble --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_model_repository -python3 $VOLUME_SRCDIR/gen_qa_models.py --ensemble --variable --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_variable_model_repository -python3 $VOLUME_SRCDIR/gen_qa_reshape_models.py --ensemble --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_reshape_model_repository -python3 $VOLUME_SRCDIR/gen_qa_identity_models.py --ensemble --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_identity_model_repository -python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --ensemble --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_sequence_model_repository -python3 $VOLUME_SRCDIR/gen_qa_sequence_models.py --ensemble --variable --models_dir=$VOLUME_ENSEMBLEDESTDIR/qa_variable_sequence_model_repository -chmod -R 777 $VOLUME_ENSEMBLEDESTDIR -python3 $VOLUME_SRCDIR/gen_tag_sigdef.py --dir $VOLUME_SIGDEFDESTDIR -chmod -R 777 $VOLUME_SIGDEFDESTDIR -python3 $VOLUME_SRCDIR/gen_qa_tf_parameters.py --models_dir $VOLUME_TFPARAMETERSDESTDIR -chmod -R 777 $VOLUME_TFPARAMETERSDESTDIR -EOF - -chmod a+x $TFSCRIPT -if [ $? -ne 0 ]; then - echo -e "Failed: chmod" - exit 1 -fi - -docker cp $TFSCRIPT $DOCKER_VOLUME_CONTAINER:$VOLUME_SRCDIR - -docker pull $TENSORFLOW_IMAGE - -echo -e "\033[34m[ INFO ] - Running: $TFSCRIPT \033[0m " - -docker run \ - --rm \ - --label RUNNER_ID=$RUNNER_ID \ - --label PROJECT_NAME=$PROJECT_NAME \ - $DOCKER_GPU_ARGS \ - -v $DOCKER_VOLUME:/mnt \ - $TENSORFLOW_IMAGE \ - bash -xe $VOLUME_SRCDIR/$TFSCRIPT - -if [ $? -ne 0 ]; then - echo -e "Failed" - exit 1 -fi - # TensorRT docker pull ${TENSORRT_IMAGE} @@ -459,7 +392,7 @@ cat > $TRTSCRIPT <_0: tag: "serve", signature_def: "serving_default", multiplier 1 - _1: tag: "serve", signature_def: , multiplier 2 - _2: tag: , signature_def: "serving_default", multiplier 3 - _3: tag: , signature_def: , multiplier 4 - - If different_io is true, there will be two variants of the model created. - The variants will have different numbers of inputs and outputs. - Alternate naming convention and config: - 0: tag: "serve", signature_def: "serving_default", two inputs/outputs - 1: tag: , signature_def: , one input/output - """ - model_version_dir = models_dir + "/" + model_name + "/" + str(model_version) - - try: - os.makedirs(model_version_dir) - except OSError as ex: - pass # ignore existing dir - - with tf.Session() as sess: - input_tensor = tf.placeholder(tf.float32, [dims], "TENSOR_INPUT") - - # tag:"serve", signature_def:"serving_default" - multiplier_0 = tf.constant(1.0, name="multiplier_0") - # tag:"serve", signature_def:signature_def_name - multiplier_1 = tf.constant(2.0, name="multiplier_1") - # tag:tag_name, signature_def:"serving_default" - multiplier_2 = tf.constant(3.0, name="multiplier_2") - # tag:tag_name, signature_def:signature_def_name - multiplier_3 = tf.constant(4.0, name="multiplier_3") - - output_tensor_0 = tf.multiply(multiplier_0, input_tensor, name="TENSOR_OUTPUT") - output_tensor_1 = tf.multiply(multiplier_1, input_tensor, name="TENSOR_OUTPUT") - output_tensor_2 = tf.multiply(multiplier_2, input_tensor, name="TENSOR_OUTPUT") - output_tensor_3 = tf.multiply(multiplier_3, input_tensor, name="TENSOR_OUTPUT") - - # build_tensor_info_op could be used if build_tensor_info is deprecated - input_tensor_info = tf.saved_model.utils.build_tensor_info(input_tensor) - output_tensor_info_0 = tf.saved_model.utils.build_tensor_info(output_tensor_0) - output_tensor_info_1 = tf.saved_model.utils.build_tensor_info(output_tensor_1) - output_tensor_info_2 = tf.saved_model.utils.build_tensor_info(output_tensor_2) - output_tensor_info_3 = tf.saved_model.utils.build_tensor_info(output_tensor_3) - - # Using predict method name because simple save uses it - # tag:"serve", signature_def:"serving_default" - signature_0 = tf.saved_model.signature_def_utils.build_signature_def( - inputs={"INPUT": input_tensor_info}, - outputs={"OUTPUT": output_tensor_info_0}, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, - ) - # tag:"serve", signature_def:signature_def_name - signature_1 = tf.saved_model.signature_def_utils.build_signature_def( - inputs={"INPUT": input_tensor_info}, - outputs={"OUTPUT": output_tensor_info_1}, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, - ) - # tag:tag_name, signature_def:"serving_default" - signature_2 = tf.saved_model.signature_def_utils.build_signature_def( - inputs={"INPUT": input_tensor_info}, - outputs={"OUTPUT": output_tensor_info_2}, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, - ) - # tag:tag_name, signature_def:signature_def_name - signature_3 = tf.saved_model.signature_def_utils.build_signature_def( - inputs={"INPUT": input_tensor_info}, - outputs={"OUTPUT": output_tensor_info_3}, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, - ) - # tag:tag_name, signature_def:signature_def_name, two inputs/outputs - signature_4 = tf.saved_model.signature_def_utils.build_signature_def( - inputs={"INPUT": input_tensor_info, "INPUT1": input_tensor_info}, - outputs={"OUTPUT": output_tensor_info_0, "OUTPUT1": output_tensor_info_1}, - method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME, - ) - - b = builder.SavedModelBuilder(model_version_dir + "/model.savedmodel") - - if different_io: - b.add_meta_graph_and_variables( - sess, - tags=[tag_name], - signature_def_map={signature_def_name: signature_0}, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - clear_devices=True, - ) - b.add_meta_graph( - tags=[tag_constants.SERVING], - signature_def_map={ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_4 - }, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - clear_devices=True, - ) - else: - signature_def_map_0 = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_0, - signature_def_name: signature_1, - } - signature_def_map_1 = { - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature_2, - signature_def_name: signature_3, - } - - b.add_meta_graph_and_variables( - sess, - tags=[tag_constants.SERVING], - signature_def_map=signature_def_map_0, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - clear_devices=True, - ) - b.add_meta_graph( - tags=[tag_name], - signature_def_map=signature_def_map_1, - assets_collection=ops.get_collection(ops.GraphKeys.ASSET_FILEPATHS), - clear_devices=True, - ) - - b.save() - - -def create_savedmodel_modelconfig( - models_dir, - model_version=1, - dims=16, - model_name="sig_tag", - tag_name="testTag", - signature_def_name="testSigDef", -): - config_dir = models_dir + "/" + model_name - config = """ -name: "{}" -platform: "tensorflow_savedmodel" -input [ - {{ - name: "INPUT" - data_type: {} - dims: [ {} ] - }} -] -output [ - {{ - name: "OUTPUT" - data_type: {} - dims: [ {} ] - }} -] -parameters: {{ -key: "TF_GRAPH_TAG" -value: {{ -string_value: "{}" -}} -}} -parameters: {{ -key: "TF_SIGNATURE_DEF" -value: {{ -string_value: "{}" -}} -}} -""".format( - model_name, - gu.np_to_model_dtype(tf.float32), - str(dims), - gu.np_to_model_dtype(tf.float32), - str(dims), - tag_name, - signature_def_name, - ) - - try: - os.makedirs(config_dir) - except OSError as ex: - pass # ignore existing dir - - with open(config_dir + "/config.pbtxt", "w") as cfile: - cfile.write(config) - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser(description="getting model output dir") - parser.add_argument("--dir", help="directory to run model in", required=True) - args = parser.parse_args() - base_dir = args.dir - base_model_name = "sig_tag" - base_tag = "serve" - test_tag = "testTag" - base_sig_def = "serving_default" - test_sig_def = "testSigDef" - - for i in range(4): - model_name = base_model_name + str(i) - create_savedmodel( - base_dir, - model_name=model_name, - tag_name=test_tag, - signature_def_name=test_sig_def, - ) - create_savedmodel( - base_dir, - model_name=base_model_name + "_different_io", - tag_name=test_tag, - signature_def_name=test_sig_def, - different_io=True, - ) - create_savedmodel_modelconfig( - base_dir, - model_name="sig_tag0", - tag_name=base_tag, - signature_def_name=base_sig_def, - ) - create_savedmodel_modelconfig( - base_dir, - model_name="sig_tag1", - tag_name=base_tag, - signature_def_name=test_sig_def, - ) - create_savedmodel_modelconfig( - base_dir, - model_name="sig_tag2", - tag_name=test_tag, - signature_def_name=base_sig_def, - ) - create_savedmodel_modelconfig( - base_dir, - model_name="sig_tag3", - tag_name=test_tag, - signature_def_name=test_sig_def, - ) - create_savedmodel_modelconfig( - base_dir, - model_name="sig_tag_different_io", - tag_name=test_tag, - signature_def_name=test_sig_def, - ) diff --git a/qa/common/infer_test.py b/qa/common/infer_test.py index 21cf630e39..aa06197373 100755 --- a/qa/common/infer_test.py +++ b/qa/common/infer_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -40,7 +40,7 @@ np_dtype_string = np.dtype(object) # Allow caller to setup specific set of backends to test -DEFAULT_BACKENDS = "graphdef savedmodel plan onnx libtorch" +DEFAULT_BACKENDS = "plan onnx libtorch" TEST_BACKENDS = os.environ.get("BACKENDS", DEFAULT_BACKENDS).split() @@ -90,29 +90,6 @@ def _infer_exact_helper( input_size = 16 - if tu.validate_for_tf_model( - input_dtype, - output0_dtype, - output1_dtype, - (input_size,), - (input_size,), - (input_size,), - ): - for pf in ["graphdef", "savedmodel"]: - if pf in TEST_BACKENDS: - _infer_exact_helper( - self, - pf, - (input_size,), - 8, - input_dtype, - output0_dtype, - output1_dtype, - output0_raw=output0_raw, - output1_raw=output1_raw, - swap=swap, - ) - if tu.validate_for_trt_model( input_dtype, output0_dtype, diff --git a/qa/common/inferentia_perf_analyzer_input_data_json/simple_model.py b/qa/common/inferentia_perf_analyzer_input_data_json/simple_model.py index db7ca95848..e9d61c9b88 100755 --- a/qa/common/inferentia_perf_analyzer_input_data_json/simple_model.py +++ b/qa/common/inferentia_perf_analyzer_input_data_json/simple_model.py @@ -1,129 +1,83 @@ -#!/usr/bin/env python -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import argparse - - -def gen_pytorch_model(name, batch_size): - class PyAddSubNet(nn.Module): - """ - Simple AddSub network in PyTorch. This network outputs the sum and - subtraction of the inputs. - """ - - def __init__(self): - super(PyAddSubNet, self).__init__() - - def forward(self, input0, input1): - return torch.sub(input0, input1, alpha=-1), torch.sub( - input0, input1, alpha=1 - ) - - model = PyAddSubNet() - model.eval() - batch_size = 1 - example_inputs = torch.zeros([8, 4], dtype=torch.int64), torch.zeros( - [8, 4], dtype=torch.int64 - ) - model_neuron = torch_neuron.trace(model, example_inputs, dynamic_batch_size=True) - model_neuron.save("{}.pt".format(name)) - - -def gen_tf_model(name, batch_size, tf_version): - # Set up model directory - model_dir = "add_sub_model" - compiled_model_dir = name - shutil.rmtree(model_dir, ignore_errors=True) - shutil.rmtree(compiled_model_dir, ignore_errors=True) - if tf_version == 1: - with tf.Session() as sess: - # Export SavedModel - input0 = tf.placeholder(tf.int64, [None, 4], "INPUT__0") - input1 = tf.placeholder(tf.int64, [None, 4], "INPUT__1") - output0 = tf.add(input0, input1, "OUTPUT__0") - output1 = tf.subtract(input0, input1, "OUTPUT__1") - tf.compat.v1.saved_model.simple_save( - session=sess, - export_dir=model_dir, - inputs={"INPUT__0": input0, "INPUT__1": input1}, - outputs={"OUTPUT__0": output0, "OUTPUT__1": output1}, - ) - # Compile using Neuron - tfn.saved_model.compile( - model_dir, - compiled_model_dir, - batch_size=batch_size, - dynamic_batch_size=True, - ) - elif tf_version == 2: - # TODO: Add gen scripts for TF2 - raise Exception("TensorFlow2 not yet supported") - else: - raise Exception("Unrecognized Tensorflow version: {}".format(tf_version)) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--model_type", - type=str, - required=True, - choices=["pytorch", "tensorflow"], - help="""The type of the compiled model. Currently, - only supports \"pytorch\" and \"tensorflow\".""", - ) - parser.add_argument( - "--name", type=str, required=True, help="The name of the compiled model" - ) - parser.add_argument( - "--tf_version", - type=int, - choices=[1, 2], - help="Version of tensorflow for compiled model", - ) - parser.add_argument( - "--batch_size", - type=int, - default=1, - help="The batch size for the compiled model", - ) - - FLAGS, unparsed = parser.parse_known_args() - if len(unparsed) > 0: - raise Exception("Unrecognized options: {}".format(unparsed)) - if FLAGS.model_type == "tensorflow": - import shutil - - import tensorflow as tf - import tensorflow.neuron as tfn - - gen_tf_model(FLAGS.name, FLAGS.batch_size, FLAGS.tf_version) - elif FLAGS.model_type == "pytorch": - import torch - import torch_neuron - from torch import nn - - gen_pytorch_model(FLAGS.name, FLAGS.batch_size) +#!/usr/bin/env python +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import argparse + + +def gen_pytorch_model(name, batch_size): + class PyAddSubNet(nn.Module): + """ + Simple AddSub network in PyTorch. This network outputs the sum and + subtraction of the inputs. + """ + + def __init__(self): + super(PyAddSubNet, self).__init__() + + def forward(self, input0, input1): + return torch.sub(input0, input1, alpha=-1), torch.sub( + input0, input1, alpha=1 + ) + + model = PyAddSubNet() + model.eval() + batch_size = 1 + example_inputs = torch.zeros([8, 4], dtype=torch.int64), torch.zeros( + [8, 4], dtype=torch.int64 + ) + model_neuron = torch_neuron.trace(model, example_inputs, dynamic_batch_size=True) + model_neuron.save("{}.pt".format(name)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_type", + type=str, + required=True, + choices=["pytorch"], + help="""The type of the compiled model. Currently, + only supports \"pytorch\".""", + ) + parser.add_argument( + "--name", type=str, required=True, help="The name of the compiled model" + ) + parser.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for the compiled model", + ) + + FLAGS, unparsed = parser.parse_known_args() + if len(unparsed) > 0: + raise Exception("Unrecognized options: {}".format(unparsed)) + elif FLAGS.model_type == "pytorch": + import torch + import torch_neuron + from torch import nn + + gen_pytorch_model(FLAGS.name, FLAGS.batch_size) diff --git a/qa/common/orca_header_test.py b/qa/common/orca_header_test.py new file mode 100755 index 0000000000..660b025556 --- /dev/null +++ b/qa/common/orca_header_test.py @@ -0,0 +1,167 @@ +#!/usr/bin/python3 +# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import sys + +sys.path.append("../common") + +import argparse +import json + +import requests + + +# To run the test, have tritonserver running and run this script with the endpoint as a flag. +# +# Example: +# ``` +# python3 orca_header_test.py http://localhost:8000/v2/models/ensemble/generate +# ``` +def get_endpoint_header(url, data, request_header=None): + """ + Sends a POST request to the given URL with the provided data and returns the value of the "endpoint-load-metrics" header, + or None if the request fails. + """ + HEADER_KEY = "endpoint-load-metrics" + try: + response = None + if request_header: + response = requests.post(url, json=data, headers=request_header) + else: + response = requests.post(url, json=data) + response.raise_for_status() + return response.headers.get(HEADER_KEY, "") + except requests.exceptions.RequestException as e: + print(f"Error making request: {e}") + return None + + +def parse_header_data(header, orca_format): + """ + Parses the header data into a dictionary based on the given format. + """ + METRIC_KEY = "named_metrics" + try: + if orca_format == "json": + # Parse the header in JSON format + data = json.loads(header.replace("JSON ", "")) + if METRIC_KEY in data: + return data[METRIC_KEY] + else: + print(f"No key '{METRIC_KEY}' in header data: {data}") + return None + elif orca_format == "text": + # Parse the header in TEXT format + data = {} + for key_value_pair in header.replace("TEXT ", "").split(", "): + key, value = key_value_pair.split("=") + if "." in key: + prefix, nested_key = key.split(".", 1) + if prefix == METRIC_KEY: + data[nested_key] = float(value) + if not data: + print(f"Could not parse any keys from header: {header}") + return None + return data + else: + print(f"Invalid ORCA format: {orca_format}") + return None + except (json.JSONDecodeError, ValueError, KeyError): + print("Error: Invalid data in the header.") + return None + + +def check_for_keys(data, desired_keys, orca_format): + """ + Checks if all desired keys are present in the given data dictionary. + """ + if all(key in data for key in desired_keys): + print( + "ORCA header present in ", + orca_format, + "format with" "kv_cache_utilization:", + [k + ": " + str(data[k]) for k in desired_keys], + ) + return True + else: + print(f"Missing keys in header: {', '.join(set(desired_keys) - set(data))}") + return False + + +def request_header(orca_format): + return {"endpoint-load-metrics-format": orca_format} if orca_format else None + + +def test_header_type(url, data, orca_format): + req_header = request_header(orca_format) + response_header = get_endpoint_header(args.url, TEST_DATA, req_header) + + desired_keys = { + "kv_cache_utilization", + "max_token_capacity", + } # Just the keys, no need to initialize with None + + if response_header is None: + print(f"Request to endpoint: '{args.url}' failed.") + return False + elif response_header == "": + if orca_format: + print( + f"response header empty, endpoint-load-metrics-format={orca_format} is not a valid ORCA metric format" + ) + return False + else: + # No request header set <=> no response header. Intended behavior. + print(f"response header empty, endpoint-load-metrics-format is not set") + return True + + data = parse_header_data(response_header, orca_format) + if data: + return check_for_keys(data, desired_keys, orca_format) + else: + print(f"Unexpected response header value: {response_header}") + return False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Make a POST request to generate endpoint to test the ORCA metrics header." + ) + parser.add_argument("url", help="The model URL to send the request to.") + args = parser.parse_args() + TEST_DATA = json.loads( + '{"text_input": "hello world", "max_tokens": 20, "bad_words": "", "stop_words": ""}' + ) + passed = True + + for format in ["json", "text", None]: + print("Checking response header for ORCA format:", format) + if not test_header_type(args.url, TEST_DATA, format): + print("FAIL on format:", format) + passed = False + + sys.exit(0 if passed else 1) diff --git a/qa/common/sequence_util.py b/qa/common/sequence_util.py index 1b2560538d..0169369c37 100755 --- a/qa/common/sequence_util.py +++ b/qa/common/sequence_util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -464,9 +464,7 @@ def check_sequence( """ if ( - ("savedmodel" not in trial) - and ("graphdef" not in trial) - and ("custom" not in trial) + ("custom" not in trial) and ("onnx" not in trial) and ("libtorch" not in trial) and ("plan" not in trial) @@ -754,9 +752,7 @@ def check_sequence_async( """ if ( - ("savedmodel" not in trial) - and ("graphdef" not in trial) - and ("custom" not in trial) + ("custom" not in trial) and ("onnx" not in trial) and ("libtorch" not in trial) and ("plan" not in trial) diff --git a/qa/common/test_util.py b/qa/common/test_util.py index d241f5909b..f3bbcdb16f 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,10 +54,6 @@ def shape_is_fixed(shape): return shape_element_count(shape) != -1 -def shape_to_tf_shape(shape): - return [None if i == -1 else i for i in shape] - - def shape_to_onnx_shape(shape, idx=0, increment_index=True): # Onnx use string for variable size dimension, and the same string # will be inferred to have same value for the model run. @@ -78,31 +74,6 @@ def shape_to_dims_str(shape): return ",".join(str(i) for i in shape) -def validate_for_tf_model( - input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape -): - """Return True if input and output dtypes are supported by a TF model.""" - - # Not extending test to uint8 yet - if ( - input_dtype == np.uint8 - or output0_dtype == np.uint8 - or output1_dtype == np.uint8 - ): - return False - - # If the input type is string the output type must be string or - # int32. This is because the QA models we generate convert strings - # internally to int32 for compute. - if (input_dtype == np.object_) and ( - ((output0_dtype != np.object_) and (output0_dtype != np.int32)) - or ((output1_dtype != np.object_) and (output1_dtype != np.int32)) - ): - return False - - return True - - def validate_for_trt_model( input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape ): diff --git a/qa/ensemble_models/batch_to_nobatch_float32_float32_float32/config.pbtxt b/qa/ensemble_models/batch_to_nobatch_float32_float32_float32/config.pbtxt index 6e95e7bc0a..383f282663 100644 --- a/qa/ensemble_models/batch_to_nobatch_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/batch_to_nobatch_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,7 +55,7 @@ ensemble_scheduling { step [ { # batch model - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/batch_to_nobatch_nobatch_float32_float32_float32/config.pbtxt b/qa/ensemble_models/batch_to_nobatch_nobatch_float32_float32_float32/config.pbtxt index ea8d70c1ac..cbc79d1def 100644 --- a/qa/ensemble_models/batch_to_nobatch_nobatch_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/batch_to_nobatch_nobatch_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,7 +55,7 @@ ensemble_scheduling { step [ { # batch model - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/mix_ensemble_int32_float32_float32/config.pbtxt b/qa/ensemble_models/mix_ensemble_int32_float32_float32/config.pbtxt index b60a3db600..e4acd279e9 100644 --- a/qa/ensemble_models/mix_ensemble_int32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/mix_ensemble_int32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ output [ ensemble_scheduling { step [ { - model_name: "graphdef_int32_int32_int32" + model_name: "onnx_int32_int32_int32" model_version: 1 input_map { key: "INPUT0" @@ -70,7 +70,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_object_object" + model_name: "onnx_int32_object_object" model_version: 1 input_map { key: "INPUT0" @@ -86,7 +86,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_object_object" + model_name: "onnx_int32_object_object" model_version: 1 input_map { key: "INPUT0" @@ -102,7 +102,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_object_int32_int32" + model_name: "onnx_object_int32_int32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/mix_nobatch_batch_float32_float32_float32/config.pbtxt b/qa/ensemble_models/mix_nobatch_batch_float32_float32_float32/config.pbtxt index 703b07a28e..a8f219fda7 100644 --- a/qa/ensemble_models/mix_nobatch_batch_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/mix_nobatch_batch_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -81,7 +81,7 @@ ensemble_scheduling { }, { # batch model - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/mix_platform_float32_float32_float32/config.pbtxt b/qa/ensemble_models/mix_platform_float32_float32_float32/config.pbtxt index 896d01fd89..c8a423a002 100644 --- a/qa/ensemble_models/mix_platform_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/mix_platform_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ output [ ensemble_scheduling { step [ { - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -70,7 +70,7 @@ ensemble_scheduling { } }, { - model_name: "savedmodel_float32_float32_float32" + model_name: "libtorch_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -81,12 +81,12 @@ ensemble_scheduling { value: "INPUT1" } output_map { - key: "OUTPUT0" + key: "OUTPUT__0" value: "double_input1" } }, { - model_name: "savedmodel_float32_float32_float32" + model_name: "libtorch_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -97,12 +97,12 @@ ensemble_scheduling { value: "INPUT0" } output_map { - key: "OUTPUT1" + key: "OUTPUT__1" value: "input0_val" } }, { - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -118,7 +118,7 @@ ensemble_scheduling { } }, { - model_name: "savedmodel_float32_float32_float32" + model_name: "libtorch_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -129,11 +129,11 @@ ensemble_scheduling { value: "input1_val" } output_map { - key: "OUTPUT0" + key: "OUTPUT__0" value: "OUTPUT0" } output_map { - key: "OUTPUT1" + key: "OUTPUT__1" value: "OUTPUT1" } } diff --git a/qa/ensemble_models/mix_type_int32_float32_float32/config.pbtxt b/qa/ensemble_models/mix_type_int32_float32_float32/config.pbtxt index 125d0977c2..caae84fe6e 100644 --- a/qa/ensemble_models/mix_type_int32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/mix_type_int32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,7 +54,7 @@ output [ ensemble_scheduling { step [ { - model_name: "graphdef_int32_int32_int32" + model_name: "onnx_int32_int32_int32" model_version: 1 input_map { key: "INPUT0" @@ -70,7 +70,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_object_object" + model_name: "onnx_int32_object_object" model_version: 1 input_map { key: "INPUT0" @@ -86,7 +86,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_object_object" + model_name: "onnx_int32_object_object" model_version: 1 input_map { key: "INPUT0" @@ -102,7 +102,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_object_int32_int32" + model_name: "onnx_object_int32_int32" model_version: 1 input_map { key: "INPUT0" @@ -118,7 +118,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_float32_float32" + model_name: "onnx_int32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -134,7 +134,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_int32_float32_float32" + model_name: "onnx_int32_float32_float32" model_version: 1 input_map { key: "INPUT0" @@ -150,7 +150,7 @@ ensemble_scheduling { } }, { - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/nobatch_to_batch_float32_float32_float32/config.pbtxt b/qa/ensemble_models/nobatch_to_batch_float32_float32_float32/config.pbtxt index 85a78ea2a0..a5886c51c6 100644 --- a/qa/ensemble_models/nobatch_to_batch_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/nobatch_to_batch_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -81,7 +81,7 @@ ensemble_scheduling { }, { # batch model - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/nobatch_to_batch_nobatch_float32_float32_float32/config.pbtxt b/qa/ensemble_models/nobatch_to_batch_nobatch_float32_float32_float32/config.pbtxt index a791296255..02ab0d1b62 100644 --- a/qa/ensemble_models/nobatch_to_batch_nobatch_float32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/nobatch_to_batch_nobatch_float32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -81,7 +81,7 @@ ensemble_scheduling { }, { # batch model - model_name: "graphdef_float32_float32_float32" + model_name: "onnx_float32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/ensemble_models/wrong_label_int32_float32_float32/config.pbtxt b/qa/ensemble_models/wrong_label_int32_float32_float32/config.pbtxt index c262211a98..1e80730dc4 100644 --- a/qa/ensemble_models/wrong_label_int32_float32_float32/config.pbtxt +++ b/qa/ensemble_models/wrong_label_int32_float32_float32/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2025, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -55,7 +55,7 @@ output [ ensemble_scheduling { step [ { - model_name: "graphdef_int32_float32_float32" + model_name: "onnx_int32_float32_float32" model_version: 1 input_map { key: "INPUT0" diff --git a/qa/python_models/python_version/model.py b/qa/python_models/python_version/model.py index 5d77906fa9..b1157ea50d 100644 --- a/qa/python_models/python_version/model.py +++ b/qa/python_models/python_version/model.py @@ -1,4 +1,4 @@ -# Copyright 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2021-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -45,16 +45,16 @@ def auto_complete_config(auto_complete_model_config): return auto_complete_model_config def initialize(self, args): - import tensorflow + import torch self.model_config = args["model_config"] # This is to make sure that /bin/bash is not picking up - # the wrong shared libraries after installing Tensorflow. + # the wrong shared libraries after installing PyTorch. # Tensorflow uses a shared library which is common with # bash. os.system("/bin/bash --help") print( - f"Python version is {sys.version_info.major}.{sys.version_info.minor}, NumPy version is {np.version.version}, and Tensorflow version is {tensorflow.__version__}", + f"Python version is {sys.version_info.major}.{sys.version_info.minor}, NumPy version is {np.version.version}, and PyTorch version is {torch.__version__}", flush=True, ) print(f"Locale is {locale.getlocale()}", flush=True) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 600d1bac92..9445464ebc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -90,6 +90,19 @@ endif() # find_package(re2 REQUIRED) +option(TRITON_ENABLE_MYSQL_ODBC "Build tritonserver with MySQL ODBC connection pool (requires ODBC::ODBC / unixODBC)" OFF) +if(TRITON_ENABLE_MYSQL_ODBC) + if(UNIX AND NOT APPLE AND NOT WIN32) + message( + STATUS + "TRITON_ENABLE_MYSQL_ODBC: ensure unixODBC development packages are installed " + "(e.g. Debian/Ubuntu: unixodbc-dev; RHEL: unixODBC-devel) so CMake can find ODBC." + ) + endif() + find_package(ODBC REQUIRED) + message(STATUS "Using ODBC ${ODBC_VERSION}") +endif() + # # tritonserver executable # @@ -101,8 +114,10 @@ add_executable( main.cc shared_memory_manager.cc triton_signal.cc + database_config.cc classification.h common.h + database_config.h shared_memory_manager.h triton_signal.h ) @@ -155,6 +170,17 @@ else() ) endif() +if(TRITON_ENABLE_MYSQL_ODBC) + target_sources( + main + PRIVATE + mysql_odbc_connection_pool.cc + mysql_odbc_connection_pool.h + ) + target_link_libraries(main PRIVATE ODBC::ODBC) + target_compile_definitions(main PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) +endif() + set(LIB_DIR "lib") if(LINUX) file(STRINGS "/etc/os-release" DISTRO_ID_LIKE REGEX "ID_LIKE") @@ -180,6 +206,7 @@ target_link_libraries( triton-common-async-work-queue # from repo-common triton-common-error # from repo-common triton-common-logging # from repo-common + triton-common-json # from repo-common (RapidJSON) triton-core-serverapi # from repo-core triton-core-serverstub # from repo-core ) @@ -337,10 +364,14 @@ if(${TRITON_ENABLE_HTTP} list(APPEND HTTP_ENDPOINT_SRCS http_server.cc + multi_infer.cc + orca_http.cc ) list(APPEND HTTP_ENDPOINT_HDRS http_server.h + http_server_macros.h + orca_http.h ) # Add header / src files based on HTTP related endpoint requested @@ -737,6 +768,61 @@ if (NOT WIN32) RUNTIME DESTINATION bin ) + # + # transform (JSON -> RapidJSON, TRITONSERVER_Error reporting) + # + add_library(transform STATIC transform.cc transform.h) + + # Required when transform is linked into shared objects (e.g. py-bindings -> + # http-endpoint-library -> transform); otherwise relocations fail at link time. + set_target_properties( + transform + PROPERTIES + POSITION_INDEPENDENT_CODE ON + ) + + target_compile_features(transform PRIVATE cxx_std_${TRITON_MIN_CXX_STANDARD}) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options( + transform + PRIVATE + /W1 /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor + ) + else() + target_compile_options( + transform + PRIVATE + -Wall -Wextra -Wno-type-limits -Wno-unused-parameter -Wno-deprecated-declarations -Werror + ) + endif() + + target_link_libraries( + transform + PRIVATE + triton-common-async-work-queue # from repo-common + triton-common-error # from repo-common + triton-common-json # RapidJSON from repo-common + triton-core-serverapi # from repo-core + triton-core-serverstub # from repo-core + ) + + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions(transform PRIVATE TRITON_ENABLE_MYSQL_ODBC=1) + target_link_libraries(transform PRIVATE ODBC::ODBC) + endif() + + if((${TRITON_ENABLE_HTTP} OR ${TRITON_ENABLE_METRICS} OR + ${TRITON_ENABLE_SAGEMAKER} OR ${TRITON_ENABLE_VERTEX_AI}) AND + TARGET http-endpoint-library AND TARGET transform) + target_link_libraries(http-endpoint-library PRIVATE transform) + if(TRITON_ENABLE_MYSQL_ODBC) + target_compile_definitions( + http-endpoint-library + PRIVATE TRITON_ENABLE_MYSQL_ODBC=1 + ) + endif() + endif() + if(${TRITON_ENABLE_GPU}) # # memory_alloc example diff --git a/src/database_config.cc b/src/database_config.cc new file mode 100644 index 0000000000..cb312dec5b --- /dev/null +++ b/src/database_config.cc @@ -0,0 +1,204 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "database_config.h" + +#include +#include + +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::optional ReadEntireFile(const std::string& path, std::string* contents) +{ + std::ifstream in(path, std::ios::binary); + if (!in) { + return std::string("failed to open file: ") + path; + } + std::ostringstream ss; + ss << in.rdbuf(); + if (!in && !in.eof()) { + return std::string("failed to read file: ") + path; + } + *contents = ss.str(); + return std::nullopt; +} + +std::optional ExpectString(const rapidjson::Value& obj, const char* key, std::string* field, bool required) +{ + if (!obj.HasMember(key)) { + if (required) { + return std::string("missing required JSON field: ") + key; + } + field->clear(); + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsString()) { + return std::string("JSON field '") + key + "' must be a string"; + } + *field = v.GetString(); + return std::nullopt; +} + +std::optional OptionalInt(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (!v.IsInt()) { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = v.GetInt(); + return std::nullopt; +} + +std::optional OptionalIntFlexible(const rapidjson::Value& obj, const char* key, int* out, int def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + int n = 0; + if (v.IsInt()) { + n = v.GetInt(); + } else if (v.IsUint()) { + if (v.GetUint() > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v.GetUint()); + } else if (v.IsInt64()) { + const int64_t v64 = v.GetInt64(); + if (v64 < INT_MIN || v64 > INT_MAX) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else if (v.IsUint64()) { + const uint64_t v64 = v.GetUint64(); + if (v64 > static_cast(INT_MAX)) { + return std::string("JSON field '") + key + "' is out of range"; + } + n = static_cast(v64); + } else { + return std::string("JSON field '") + key + "' must be an integer"; + } + *out = n; + return std::nullopt; +} + +std::optional OptionalNonNegativeSize( + const rapidjson::Value& obj, const char* key, std::size_t* out, + std::size_t def) +{ + if (!obj.HasMember(key)) { + *out = def; + return std::nullopt; + } + const auto& v = obj[key]; + if (v.IsUint64()) { + *out = static_cast(v.GetUint64()); + return std::nullopt; + } + if (v.IsInt64()) { + const int64_t n = v.GetInt64(); + if (n < 0) { + return std::string("JSON field '") + key + "' must be non-negative"; + } + *out = static_cast(n); + return std::nullopt; + } + return std::string("JSON field '") + key + "' must be an integer"; +} + +} // namespace + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out) +{ + std::string raw; + if (auto e = ReadEntireFile(path, &raw)) { + return e; + } + + rapidjson::Document doc; + doc.Parse(raw.c_str()); + if (doc.HasParseError()) { + return std::string("JSON parse error: ") +rapidjson::GetParseError_En(doc.GetParseError()) + " at offset " + std::to_string(doc.GetErrorOffset()); + } + if (!doc.IsObject()) { + return std::string("root JSON value must be an object"); + } + + DatabaseConfig c; + if (auto e = ExpectString(doc, "databaseIp", &c.database_ip, false)) return e; + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.front()))) c.database_ip.erase(0, 1); + + while (!c.database_ip.empty() && std::isspace(static_cast(c.database_ip.back()))) c.database_ip.pop_back(); + + if (auto e = OptionalIntFlexible(doc, "databasePort", &c.database_port, 3306)) return e; + + if (c.database_port < 1 || c.database_port > 65535) return std::string("databasePort must be between 1 and 65535"); + + if (auto e = ExpectString(doc, "odbcDriverName", &c.odbc_driver_name, false)) return e; + + if (auto e = ExpectString(doc, "primaryDSNName", &c.primary_dsn_name, true)) return e; + + if (auto e = ExpectString(doc, "secondaryDSNName", &c.secondary_dsn_name, false)) return e; + + if (auto e = ExpectString(doc, "dsnUserName", &c.dsn_user_name, true)) return e; + + if (auto e = ExpectString(doc, "dsnUserPassword", &c.dsn_user_password, true)) return e; + + int dc_id_int = 0; + if (auto e = OptionalIntFlexible(doc, "dcId", &dc_id_int, 0)) return e; + if (dc_id_int < INT32_MIN || dc_id_int > INT32_MAX) return std::string("dcId is out of range for int32_t"); + c.dc_id = static_cast(dc_id_int); + + if (auto e = OptionalInt(doc, "queryRetryCount", &c.query_retry_count, 3)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "minPoolConnections", &c.min_pool_connections, 2)) return e; + + if (auto e = OptionalNonNegativeSize(doc, "maxPoolConnections", &c.max_pool_connections, 5)) return e; + + if (c.min_pool_connections < 1) return std::string("minPoolConnections must be at least 1"); + + if (c.max_pool_connections < c.min_pool_connections) return std::string("maxPoolConnections must be greater than or equal to minPoolConnections"); + *out = std::move(c); + return std::nullopt; +} + +}} // namespace triton::server + diff --git a/src/database_config.h b/src/database_config.h new file mode 100644 index 0000000000..c1083ed020 --- /dev/null +++ b/src/database_config.h @@ -0,0 +1,56 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +namespace triton { namespace server { + +inline constexpr const char kTritonDmConfigJsonPath[] = "/etc/triton-dmconfig.json"; + +struct DatabaseConfig { + std::string database_ip; + int database_port{3306}; + std::string odbc_driver_name; + + std::string primary_dsn_name; + std::string secondary_dsn_name; + std::string dsn_user_name; + std::string dsn_user_password; + + int32_t dc_id{0}; + int query_retry_count{3}; + std::size_t min_pool_connections{2}; + std::size_t max_pool_connections{5}; +}; + +std::optional LoadDatabaseConfigFromJsonFile(const std::string& path, DatabaseConfig* out); + +}} // namespace triton::server + diff --git a/src/http_server.cc b/src/http_server.cc index 4f8c0453f6..845caf0569 100644 --- a/src/http_server.cc +++ b/src/http_server.cc @@ -1,4 +1,4 @@ -// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -35,16 +35,19 @@ #include #include +#include +#include #include #include - +#include +#include "triton/common/triton_json.h" #include "classification.h" +#include "http_server_macros.h" #define TRITONJSON_STATUSTYPE TRITONSERVER_Error* #define TRITONJSON_STATUSRETURN(M) \ return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, (M).c_str()) #define TRITONJSON_STATUSSUCCESS nullptr -#include "triton/common/triton_json.h" namespace triton { namespace server { @@ -58,36 +61,6 @@ namespace triton { namespace server { } \ } while (false) -#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ - do { \ - TRITONSERVER_Error* err__ = (X); \ - if (err__ != nullptr) { \ - EVBufferAddErrorJson((REQ)->buffer_out, err__); \ - evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ - TRITONSERVER_ErrorDelete(err__); \ - return; \ - } \ - } while (false) - -#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ - do { \ - EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ - evhtp_send_reply((REQ), CODE); \ - return; \ - } while (false) - -#define RETURN_AND_RESPOND_IF_RESTRICTED( \ - REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ - do { \ - auto const& is_restricted_api = \ - RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ - auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ - if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ - return; \ - } \ - } while (false) - - namespace { int @@ -3226,6 +3199,27 @@ HTTPAPIServer::ForwardHeaders( return nullptr; // success } +TRITONSERVER_Error* +HTTPAPIServer::ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse* response, const uint32_t flags, + void* userp)) +{ + RETURN_IF_ERR(ForwardHeaders(req, irequest)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetReleaseCallback( + irequest, InferRequestClass::InferRequestComplete, + request_release_payload)); + RETURN_IF_ERR(TRITONSERVER_InferenceRequestSetResponseCallback( + irequest, allocator_, + reinterpret_cast(&infer_request->alloc_payload_), + infer_response_complete_fn, reinterpret_cast(infer_request))); + return TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); +} + void HTTPAPIServer::HandleGenerate( evhtp_request_t* req, const std::string& model_name, @@ -3392,6 +3386,7 @@ HTTPAPIServer::HandleGenerate( request_release_payload.release(); } + TRITONSERVER_Error* HTTPAPIServer::ModelInputMetadata( const std::string& model_name, const int64_t model_version, @@ -3693,25 +3688,11 @@ HTTPAPIServer::HandleInfer( request_id = ""; } - RETURN_AND_CALLBACK_IF_ERR(ForwardHeaders(req, irequest), error_callback); - auto request_release_payload = std::make_unique( irequest_shared, decompressed_buffer); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetReleaseCallback( - irequest, InferRequestClass::InferRequestComplete, - request_release_payload.get()), - error_callback); - RETURN_AND_CALLBACK_IF_ERR( - TRITONSERVER_InferenceRequestSetResponseCallback( - irequest, allocator_, - reinterpret_cast(&infer_request->alloc_payload_), - InferRequestClass::InferResponseComplete, - reinterpret_cast(infer_request.get())), - error_callback); - - auto err = - TRITONSERVER_ServerInferAsync(server_.get(), irequest, triton_trace); + auto err = ScheduleInferAsync( + req, irequest, infer_request.get(), request_release_payload.get(), + triton_trace); #ifdef TRITON_ENABLE_TRACING // Ownership of trace passed to Triton core, set trace to null to mark it // as no longer owned here. @@ -3774,17 +3755,23 @@ HTTPAPIServer::InferRequestClass::InferRequestClass( TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request, bool register_fini_cancel_hook) : server_(server), req_(req), response_compression_type_(response_compression_type), response_count_(0), + register_fini_cancel_hook_(register_fini_cancel_hook), triton_request_(triton_request), shm_manager_(shm_manager) { evhtp_connection_t* htpconn = evhtp_request_get_connection(req); thread_ = htpconn->thread; - evhtp_request_pause(req); - evhtp_request_set_hook( - req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, - reinterpret_cast(this)); + if (pause_http_request) { + evhtp_request_pause(req); + } + if (register_fini_cancel_hook_) { + evhtp_request_set_hook( + req_, evhtp_hook_on_request_fini, (evhtp_hook)(void*)RequestFiniHook, + reinterpret_cast(this)); + } } void @@ -3831,7 +3818,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( std::to_string(infer_request->response_count_)) .c_str()); } else if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); #ifdef TRITON_ENABLE_TRACING if (infer_request->trace_ != nullptr) { infer_request->trace_->CaptureTimestamp( @@ -3861,7 +3848,7 @@ HTTPAPIServer::InferRequestClass::InferResponseComplete( TRITONSERVER_Error* HTTPAPIServer::InferRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -3958,6 +3945,16 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( // Handle data. SHM outputs will not have an info. auto info = reinterpret_cast(userp); + if (json_only_out != nullptr) { + if (info == nullptr || info->kind_ != AllocPayload::OutputInfo::JSON || + info->class_cnt_ > 0) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "multi_infer sub-request: only plain JSON outputs are supported (no " + "shared memory, binary tensor data, or classification)"); + } + } + size_t element_count = 1; uint32_t batch_size = 0; @@ -4095,34 +4092,45 @@ HTTPAPIServer::InferRequestClass::FinalizeResponse( } evbuffer* response_body = response_placeholder; - switch (response_compression_type_) { - case DataCompressor::Type::DEFLATE: - case DataCompressor::Type::GZIP: { - auto compressed_buffer = evbuffer_new(); - auto err = DataCompressor::CompressData( - response_compression_type_, response_placeholder, compressed_buffer); - if (err == nullptr) { - response_body = compressed_buffer; - evbuffer_free(response_placeholder); - } else { - // just log the compression error and return the uncompressed data - LOG_VERBOSE(1) << "unable to compress response: " - << TRITONSERVER_ErrorMessage(err); - TRITONSERVER_ErrorDelete(err); - evbuffer_free(compressed_buffer); - response_compression_type_ = DataCompressor::Type::IDENTITY; + if (json_only_out == nullptr) { + switch (response_compression_type_) { + case DataCompressor::Type::DEFLATE: + case DataCompressor::Type::GZIP: { + auto compressed_buffer = evbuffer_new(); + auto err = DataCompressor::CompressData( + response_compression_type_, response_placeholder, + compressed_buffer); + if (err == nullptr) { + response_body = compressed_buffer; + evbuffer_free(response_placeholder); + } else { + // just log the compression error and return the uncompressed data + LOG_VERBOSE(1) << "unable to compress response: " + << TRITONSERVER_ErrorMessage(err); + TRITONSERVER_ErrorDelete(err); + evbuffer_free(compressed_buffer); + response_compression_type_ = DataCompressor::Type::IDENTITY; + } + break; } - break; + case DataCompressor::Type::IDENTITY: + case DataCompressor::Type::UNKNOWN: + // Do nothing for other cases + break; } - case DataCompressor::Type::IDENTITY: - case DataCompressor::Type::UNKNOWN: - // Do nothing for other cases - break; + SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); + evbuffer_add_buffer(req_->buffer_out, response_body); + } else { + if (!ordered_buffers.empty()) { + evbuffer_free(response_placeholder); + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_UNSUPPORTED, + "multi_infer sub-request: binary outputs are not supported"); + } + evbuffer_add_buffer(json_only_out, response_body); } - SetResponseHeader(!ordered_buffers.empty(), buffer.Size()); - evbuffer_add_buffer(req_->buffer_out, response_body); // Destroy the evbuffer object as the data has been moved - // to HTTP response buffer + // to HTTP response buffer (or json_only_out) evbuffer_free(response_body); return nullptr; // success @@ -4193,7 +4201,7 @@ HTTPAPIServer::GenerateRequestClass::InferResponseComplete( TRITONSERVER_Error* err = nullptr; if (response != nullptr) { - err = infer_request->FinalizeResponse(response); + err = infer_request->FinalizeResponse(response, nullptr); } if (err != nullptr) { infer_request->AddErrorJson(err); @@ -4238,6 +4246,35 @@ HTTPAPIServer::GenerateRequestClass::StartResponse( return; } + +#ifdef TRITON_ENABLE_METRICS + // logic to add kv_cache metrics to response header + // Get the metrics in Prometheus format + + // ENDPOINT_LOAD_METRICS_TYPE is request header that specifies which load + // report format `endpoint-load-metrics` will be in. If not present, the + // response header will not be written and the feature is disabled. + // + // The valid values for ENDPOINT_LOAD_METRICS_TYPE header are: + // + // "text" + // "json" + // + // Any other value will have behavior equivalent to being unset while also + // logging an error. + auto server = infer_request->EvHtpServer(); + const char* orca_metric_format = nullptr; + evhtp_header_t* metric_format_header = + evhtp_headers_find_header(req->headers_in, ENDPOINT_LOAD_METRICS_TYPE); + + if (metric_format_header != nullptr) { + orca_metric_format = metric_format_header->val; + } + if (orca_metric_format != nullptr && server != nullptr) { + SetEndpointLoadMetricsHeader(req, orca_metric_format, server); + } +#endif // TRITON_ENABLE_METRICS + if (infer_request->streaming_) { AddContentTypeHeader(req, "text/event-stream; charset=utf-8"); } else { @@ -4324,8 +4361,13 @@ HTTPAPIServer::GenerateRequestClass::SendChunkResponse(bool end) TRITONSERVER_Error* HTTPAPIServer::GenerateRequestClass::FinalizeResponse( - TRITONSERVER_InferenceResponse* response) + TRITONSERVER_InferenceResponse* response, evbuffer* json_only_out) { + if (json_only_out != nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INTERNAL, + "JSON-only response aggregation is not supported for generate"); + } triton_response_ = response; RETURN_IF_ERR(TRITONSERVER_InferenceResponseError(response)); @@ -4605,12 +4647,15 @@ HTTPAPIServer::GenerateRequestClass::ExactMappingOutput( return nullptr; // success } -void -HTTPAPIServer::Handle(evhtp_request_t* req) -{ - LOG_VERBOSE(1) << "HTTP request: " << req->method << " " - << req->uri->path->full; +void HTTPAPIServer::Handle(evhtp_request_t* req) { + LOG_VERBOSE(1) << "HTTP request: " << req->method << " " << req->uri->path->full; +#ifdef TRITON_ENABLE_MYSQL_ODBC + if (std::string(req->uri->path->full) == "/v2/multi_infer") { + HandleMultiInfer(req); + return; + } +#endif if (std::string(req->uri->path->full) == "/v2/models/stats") { // model statistics HandleModelStats(req); diff --git a/src/http_server.h b/src/http_server.h index efadc21675..40e3a65263 100644 --- a/src/http_server.h +++ b/src/http_server.h @@ -1,4 +1,4 @@ -// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -39,6 +39,7 @@ #include "common.h" #include "data_compressor.h" +#include "orca_http.h" #include "restricted_features.h" #include "shared_memory_manager.h" #include "tracer.h" @@ -277,11 +278,13 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager); + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true); virtual ~InferRequestClass() { - if (req_ != nullptr) { + if (req_ != nullptr && register_fini_cancel_hook_) { evhtp_request_unset_hook(req_, evhtp_hook_on_request_fini); } req_ = nullptr; @@ -318,8 +321,11 @@ class HTTPAPIServer : public HTTPServer { static void InferResponseComplete( TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp); + // When json_only_out is non-null, write infer response JSON there only + // (no HTTP headers); used by POST /v2/multi_infer aggregation. virtual TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response); + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr); // Helper function to set infer response header in the form specified by // the endpoint protocol @@ -328,6 +334,8 @@ class HTTPAPIServer : public HTTPServer { uint32_t IncrementResponseCount(); + uint32_t ResponseCount() const { return response_count_.load(); } + // Only used if tracing enabled std::shared_ptr trace_; @@ -358,6 +366,8 @@ class HTTPAPIServer : public HTTPServer { // Counter to keep track of number of responses generated. std::atomic response_count_{0}; + const bool register_fini_cancel_hook_; + // Event hook for called before request deletion static evhtp_res RequestFiniHook(evhtp_request* req, void* arg); @@ -397,6 +407,8 @@ class HTTPAPIServer : public HTTPServer { } virtual ~GenerateRequestClass(); + TRITONSERVER_Server* EvHtpServer() const { return server_; } + // [FIXME] Specialize response complete function for now, should have // been a dispatcher and call into object specific response function. static void InferResponseComplete( @@ -409,7 +421,8 @@ class HTTPAPIServer : public HTTPServer { // Response preparation TRITONSERVER_Error* FinalizeResponse( - TRITONSERVER_InferenceResponse* response) override; + TRITONSERVER_InferenceResponse* response, + evbuffer* json_only_out = nullptr) override; void AddErrorJson(TRITONSERVER_Error* error); static void StartResponse(evthr_t* thr, void* arg, void* shared); @@ -436,6 +449,7 @@ class HTTPAPIServer : public HTTPServer { // TENSOR, PARAMETER type uint32_t index; }; + TRITONSERVER_Error* ExactMappingInput( const std::string& name, triton::common::TritonJson::Value& value, std::map& @@ -488,6 +502,7 @@ class HTTPAPIServer : public HTTPServer { evbuffer* buffer_ = nullptr; }; + protected: explicit HTTPAPIServer( const std::shared_ptr& server, @@ -501,11 +516,13 @@ class HTTPAPIServer : public HTTPServer { // [FIXME] extract to "infer" class virtual std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) { return std::unique_ptr(new InferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } // Helper function to retrieve infer request header in the form specified by @@ -536,6 +553,17 @@ class HTTPAPIServer : public HTTPServer { TRITONSERVER_Error* ForwardHeaders( evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest); + // ForwardHeaders, release/response callbacks, and ServerInferAsync (shared + // by HandleInfer and multi_infer sub-requests). + TRITONSERVER_Error* ScheduleInferAsync( + evhtp_request_t* req, TRITONSERVER_InferenceRequest* irequest, + InferRequestClass* infer_request, + RequestReleasePayload* request_release_payload, + TRITONSERVER_InferenceTrace* triton_trace, + void (*infer_response_complete_fn)( + TRITONSERVER_InferenceResponse*, const uint32_t, void*) = + InferRequestClass::InferResponseComplete); + static TRITONSERVER_Error* InferResponseAlloc( TRITONSERVER_ResponseAllocator* allocator, const char* tensor_name, size_t byte_size, TRITONSERVER_MemoryType preferred_memory_type, @@ -568,6 +596,8 @@ class HTTPAPIServer : public HTTPServer { void HandleInfer( evhtp_request_t* req, const std::string& model_name, const std::string& model_version_str); + // POST /v2/multi_infer — parallel infer for multiple models in one HTTP call. + void HandleMultiInfer(evhtp_request_t* req); void HandleModelStats( evhtp_request_t* req, const std::string& model_name = "", const std::string& model_version_str = ""); diff --git a/src/http_server_macros.h b/src/http_server_macros.h new file mode 100644 index 0000000000..3cd47a1b14 --- /dev/null +++ b/src/http_server_macros.h @@ -0,0 +1,41 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Shared preprocessor macros for HTTP API handlers. Macros are not C++ symbols +// and are not "in" a namespace; this header is included from within +// triton::server for consistency with http_server.cc. +// +// Prerequisites at expansion sites: HttpCodeFromError, EVBufferAddErrorJson, +// and (for RETURN_AND_RESPOND_IF_RESTRICTED) RespondIfRestricted must be +// visible — typically from the same translation unit's anonymous namespace and +// HTTPAPIServer member functions respectively. + +#pragma once + +#define RETURN_AND_RESPOND_IF_ERR(REQ, X) \ + do { \ + TRITONSERVER_Error* err__ = (X); \ + if (err__ != nullptr) { \ + EVBufferAddErrorJson((REQ)->buffer_out, err__); \ + evhtp_send_reply((REQ), HttpCodeFromError(err__)); \ + TRITONSERVER_ErrorDelete(err__); \ + return; \ + } \ + } while (false) + +#define RETURN_AND_RESPOND_WITH_ERR(REQ, CODE, MSG) \ + do { \ + EVBufferAddErrorJson((REQ)->buffer_out, MSG); \ + evhtp_send_reply((REQ), CODE); \ + return; \ + } while (false) + +#define RETURN_AND_RESPOND_IF_RESTRICTED( \ + REQ, RESTRICTED_CATEGORY, RESTRICTED_APIS) \ + do { \ + auto const& is_restricted_api = \ + RESTRICTED_APIS.IsRestricted(RESTRICTED_CATEGORY); \ + auto const& restriction = RESTRICTED_APIS.Get(RESTRICTED_CATEGORY); \ + if (is_restricted_api && RespondIfRestricted(REQ, restriction)) { \ + return; \ + } \ + } while (false) diff --git a/src/main.cc b/src/main.cc index b2eee17c68..38b21ec648 100644 --- a/src/main.cc +++ b/src/main.cc @@ -42,11 +42,20 @@ #include #include +#include +#include +#include #include #include +#include +#include #include #include +#include "database_config.h" +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "mysql_odbc_connection_pool.h" +#endif // TRITON_ENABLE_MYSQL_ODBC #include "triton_signal.h" #ifdef TRITON_ENABLE_ASAN @@ -103,6 +112,113 @@ std::unique_ptr g_vertex_ai_service; triton::server::TritonServerParameters g_triton_params; +// Populated at startup when /etc/triton-dmconfig.json is present. +std::optional g_triton_dm_database_config; + +#ifdef TRITON_ENABLE_MYSQL_ODBC +// ODBC pool opened after successful config load (same lifetime as the process). +std::unique_ptr g_triton_dm_odbc_pool; + +// Registered with std::atexit after the pool is initialized so connections are +// released on normal return, exit(), and FAIL_IF_ERR paths. +extern "C" void TritonDmOdbcPoolAtExit(void) +{ + triton::server::SetGlobalMysqlOdbcPool(nullptr); + g_triton_dm_odbc_pool.reset(); +} + +void TritonModelsRefreshThreadMain() +{ + using std::chrono_literals::operator""min; + while (!triton::server::signal_exiting_) { + if (auto err = triton::server::UpdateTritonModelsData()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Triton ML models DB refresh failed: " << *err; +#endif // TRITON_ENABLE_LOGGING + } + std::unique_lock lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.wait_for(lock, 15min, [] { return triton::server::signal_exiting_; }); + } +} + +std::optional g_triton_models_refresh_thread; + +void StartTritonModelsRefreshThread() +{ + if (!g_triton_dm_odbc_pool) { + return; + } + g_triton_models_refresh_thread.emplace(TritonModelsRefreshThreadMain); +} + +void JoinTritonModelsRefreshThread() +{ + if (!g_triton_models_refresh_thread.has_value()) { + return; + } + if (g_triton_models_refresh_thread->joinable()) { + { + std::lock_guard lock(triton::server::signal_exit_mu_); + triton::server::signal_exit_cv_.notify_all(); + } + g_triton_models_refresh_thread->join(); + } + g_triton_models_refresh_thread.reset(); +} +#endif // TRITON_ENABLE_MYSQL_ODBC + +void LoadTritonDmDatabaseConfigAtStartup() +{ + namespace fs = std::filesystem; + const std::string path(triton::server::kTritonDmConfigJsonPath); + if (!fs::exists(path)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "Database config file '" << path << "' not found; continuing without DM database metadata."; +#else + std::cerr << "warning: database config file '" << path << "' not found; continuing without DM database metadata." << std::endl; +#endif // TRITON_ENABLE_LOGGING + return; + } + + triton::server::DatabaseConfig cfg; + if (auto err = triton::server::LoadDatabaseConfigFromJsonFile(path, &cfg)) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to load '" << path << "': " << *err; +#else + std::cerr << "Failed to load '" << path << "': " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + exit(1); + } + + g_triton_dm_database_config = std::move(cfg); + +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "Loaded database config from '" << path << "' (databaseIp='" << g_triton_dm_database_config->database_ip << "', databasePort=" << g_triton_dm_database_config->database_port << ")"; +#endif // TRITON_ENABLE_LOGGING + +#ifdef TRITON_ENABLE_MYSQL_ODBC + g_triton_dm_odbc_pool = std::make_unique(*g_triton_dm_database_config); + if (auto err = g_triton_dm_odbc_pool->Initialize()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_ERROR << "Failed to initialize MySQL ODBC connection pool: " << *err; +#else + std::cerr << "Failed to initialize MySQL ODBC connection pool: " << *err << std::endl; +#endif // TRITON_ENABLE_LOGGING + g_triton_dm_odbc_pool.reset(); + exit(1); + } + if (std::atexit(TritonDmOdbcPoolAtExit) != 0) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "std::atexit failed; ODBC pool will still be destroyed at static exit"; +#endif // TRITON_ENABLE_LOGGING + } + triton::server::SetGlobalMysqlOdbcPool(g_triton_dm_odbc_pool.get()); +#ifdef TRITON_ENABLE_LOGGING + LOG_INFO << "MySQL ODBC connection pool started ("<< g_triton_dm_database_config->max_pool_connections << " connections to DSN '" << g_triton_dm_database_config->primary_dsn_name << "')"; +#endif // TRITON_ENABLE_LOGGING +#endif // TRITON_ENABLE_MYSQL_ODBC +} + #ifdef TRITON_ENABLE_GRPC TRITONSERVER_Error* StartGrpcService( @@ -469,6 +585,8 @@ main(int argc, char** argv) LOG_SET_OUT_FILE(g_triton_params.log_file_); #endif // TRITON_ENABLE_LOGGING + LoadTritonDmDatabaseConfigAtStartup(); + // Trace manager. triton::server::TraceManager* trace_manager; @@ -501,6 +619,10 @@ main(int argc, char** argv) exit(1); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + StartTritonModelsRefreshThread(); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Wait until a signal terminates the server... while (!triton::server::signal_exiting_) { // If enabled, poll the model repository to see if there have been @@ -521,6 +643,10 @@ main(int argc, char** argv) triton::server::signal_exit_cv_.wait_for(lock, wait_timeout); } +#ifdef TRITON_ENABLE_MYSQL_ODBC + JoinTritonModelsRefreshThread(); +#endif // TRITON_ENABLE_MYSQL_ODBC + // Stop the HTTP[, gRPC, and metrics] endpoints, and update exit timeout. uint32_t exit_timeout_secs = g_triton_params.exit_timeout_secs_; StopEndpoints(&exit_timeout_secs); diff --git a/src/multi_infer.cc b/src/multi_infer.cc new file mode 100644 index 0000000000..e7659446e5 --- /dev/null +++ b/src/multi_infer.cc @@ -0,0 +1,635 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "http_server.h" + +#include "classification.h" +#include "common.h" +#include "transform.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +#include "http_server_macros.h" + +namespace { + +constexpr size_t kMaxMultiInferRequests = 16; + +int HttpCodeFromError(TRITONSERVER_Error* error) { + if (error == nullptr) { + return EVHTP_RES_OK; + } + switch (TRITONSERVER_ErrorCode(error)) { + case TRITONSERVER_ERROR_INTERNAL: + return EVHTP_RES_SERVERR; + case TRITONSERVER_ERROR_NOT_FOUND: + return EVHTP_RES_NOTFOUND; + case TRITONSERVER_ERROR_UNAVAILABLE: + return EVHTP_RES_SERVUNAVAIL; + case TRITONSERVER_ERROR_UNSUPPORTED: + return EVHTP_RES_NOTIMPL; + case TRITONSERVER_ERROR_UNKNOWN: + case TRITONSERVER_ERROR_INVALID_ARG: + case TRITONSERVER_ERROR_ALREADY_EXISTS: + case TRITONSERVER_ERROR_CANCELLED: + return EVHTP_RES_BADREQ; + } + + return EVHTP_RES_BADREQ; +} + +void EVBufferAddErrorJson(evbuffer* buffer, const char* message) { + triton::common::TritonJson::Value response(triton::common::TritonJson::ValueType::OBJECT); + response.AddStringRef("error", message, strlen(message)); + + triton::common::TritonJson::WriteBuffer buffer_json; + response.Write(&buffer_json); + + evbuffer_add(buffer, buffer_json.Base(), buffer_json.Size()); +} + +void EVBufferAddErrorJson(evbuffer* buffer, TRITONSERVER_Error* err) { + const char* message = TRITONSERVER_ErrorMessage(err); + EVBufferAddErrorJson(buffer, message); +} + +void AddContentTypeHeader(evhtp_request_t* req, const char* type) { + auto content_header = evhtp_headers_find_header(req->headers_out, kContentTypeHeader); + if (content_header) { + evhtp_header_rm_and_free(req->headers_out, content_header); + } + + evhtp_headers_add_header(req->headers_out, evhtp_header_new(kContentTypeHeader, type, 1, 1)); +} + +TRITONSERVER_Error* CopyInferSlotBodyJson(triton::common::TritonJson::Value& slot, triton::common::TritonJson::Value* infer_json) { + *infer_json = triton::common::TritonJson::Value(triton::common::TritonJson::ValueType::OBJECT); + { + triton::common::TritonJson::Value v; + if (slot.Find("id", &v)) { + RETURN_IF_ERR(infer_json->Add("id", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (!slot.Find("inputs", &v)) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Each entry in 'requests' must contain an 'inputs' array"); + } + RETURN_IF_ERR(infer_json->Add("inputs", std::move(v))); + } + { + triton::common::TritonJson::Value v; + if (slot.Find("outputs", &v)) { + RETURN_IF_ERR(infer_json->Add("outputs", std::move(v))); + } + } + { + triton::common::TritonJson::Value v; + if (slot.Find("parameters", &v)) { + RETURN_IF_ERR(infer_json->Add("parameters", std::move(v))); + } + } + return nullptr; +} + +TRITONSERVER_Error* GetModelVersionStringFromSlot(triton::common::TritonJson::Value& slot, std::string* ver_out) +{ + ver_out->clear(); + triton::common::TritonJson::Value mv; + if (!slot.Find("model_version", &mv)) { + return nullptr; + } + if (mv.IsString()) { + const char* s; + size_t len; + RETURN_IF_ERR(mv.AsString(&s, &len)); + ver_out->assign(s, len); + return nullptr; + } + if (mv.IsNumber()) { + int64_t iv; + RETURN_IF_ERR(mv.AsInt(&iv)); + *ver_out = std::to_string(iv); + return nullptr; + } + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'model_version' must be a string or integer"); +} + +class MultiInferAggregator : public std::enable_shared_from_this { + private: + struct FinishPayload { + std::shared_ptr agg; + }; + + public: + MultiInferAggregator(evhtp_request_t* req, size_t slot_count, evthr_t* reply_thread, std::vector> irequests) + : req_(req), n_(slot_count), reply_thread_(reply_thread), + irequests_(std::move(irequests)), success_json_(slot_count), + error_text_(slot_count), have_error_(slot_count, 0){} + + std::shared_ptr IrequestAt(size_t i) const + { + return irequests_[i]; + } + + void CancelAllSubRequests() + { + std::lock_guard lk(mu_); + if (cancel_sent_) { + return; + } + cancel_sent_ = true; + for (auto& ir : irequests_) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); + } + } + } + + void OnShardDone(size_t slot, TRITONSERVER_Error* finalize_err, const std::string& response_json) { + std::shared_ptr self; + { + std::lock_guard lk(mu_); + if (finalize_err != nullptr) { + have_error_[slot] = 1; + error_text_[slot] = TRITONSERVER_ErrorMessage(finalize_err); + TRITONSERVER_ErrorDelete(finalize_err); + if (!cancel_sent_) { + cancel_sent_ = true; + for (auto& ir : irequests_) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestCancel(ir.get()), "cancelling multi_infer sub-request"); + } + } + } + } else { + success_json_[slot] = response_json; + } + done_count_++; + if (done_count_ < n_ || reply_scheduled_) { + return; + } + reply_scheduled_ = true; + self = shared_from_this(); + } + + auto* fp = new FinishPayload{std::move(self)}; + evthr_defer(reply_thread_, FinishThunk, fp); + } + + private: + static void FinishThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + std::unique_ptr fp(static_cast(arg)); + fp->agg->WriteHttpReply(); + } + + void WriteHttpReply() { + triton::common::TritonJson::Value root(triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value responses(root, triton::common::TritonJson::ValueType::ARRAY); + for (size_t i = 0; i < n_; ++i) { + if (have_error_[i]) { + triton::common::TritonJson::Value item(root, triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); + TRITONSERVER_Error* ae = err_part.AddString("message", error_text_[i]); + if (ae != nullptr) { + LOG_TRITONSERVER_ERROR(ae, "multi_infer: building error JSON"); + TRITONSERVER_ErrorDelete(ae); + } + TRITONSERVER_Error* be = item.Add("error", std::move(err_part)); + if (be != nullptr) { + LOG_TRITONSERVER_ERROR(be, "multi_infer: building error JSON"); + TRITONSERVER_ErrorDelete(be); + } + TRITONSERVER_Error* ce = responses.Append(std::move(item)); + if (ce != nullptr) { + LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); + TRITONSERVER_ErrorDelete(ce); + } + } + else { + triton::common::TritonJson::Value item; + TRITONSERVER_Error* perr = item.Parse(success_json_[i].c_str(), success_json_[i].size()); + if (perr != nullptr) { + triton::common::TritonJson::Value wrap(root, triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value err_part(root, triton::common::TritonJson::ValueType::OBJECT); + TRITONSERVER_Error* ae = err_part.AddString("message", TRITONSERVER_ErrorMessage(perr)); + TRITONSERVER_ErrorDelete(perr); + if (ae != nullptr) { + TRITONSERVER_ErrorDelete(ae); + } + TRITONSERVER_Error* be = wrap.Add("error", std::move(err_part)); + if (be != nullptr) { + TRITONSERVER_ErrorDelete(be); + } + TRITONSERVER_Error* ce = responses.Append(std::move(wrap)); + if (ce != nullptr) { + TRITONSERVER_ErrorDelete(ce); + } + } else { + TRITONSERVER_Error* ce = responses.Append(std::move(item)); + if (ce != nullptr) { + LOG_TRITONSERVER_ERROR(ce, "multi_infer: appending response"); + TRITONSERVER_ErrorDelete(ce); + } + } + } + } + + TRITONSERVER_Error* re = root.Add("responses", std::move(responses)); + if (re != nullptr) { + LOG_TRITONSERVER_ERROR(re, "multi_infer: building root JSON"); + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, re); + evhtp_send_reply(req_, HttpCodeFromError(re)); + TRITONSERVER_ErrorDelete(re); + evhtp_request_resume(req_); + return; + } + + triton::common::TritonJson::WriteBuffer wb; + TRITONSERVER_Error* we = root.Write(&wb); + if (we != nullptr) { + AddContentTypeHeader(req_, "application/json"); + EVBufferAddErrorJson(req_->buffer_out, we); + evhtp_send_reply(req_, HttpCodeFromError(we)); + TRITONSERVER_ErrorDelete(we); + } + else { + AddContentTypeHeader(req_, "application/json"); + evbuffer_add(req_->buffer_out, wb.Base(), wb.Size()); + evhtp_send_reply(req_, EVHTP_RES_OK); + } + evhtp_request_resume(req_); + } + + evhtp_request_t* req_; + const size_t n_; + evthr_t* reply_thread_; + std::vector> irequests_; + + std::mutex mu_; + size_t done_count_{0}; + std::vector success_json_; + std::vector error_text_; + std::vector have_error_; + bool cancel_sent_{false}; + bool reply_scheduled_{false}; +}; + +class MultiInferShardRequest : public HTTPAPIServer::InferRequestClass { + public: + MultiInferShardRequest(TRITONSERVER_Server* server, evhtp_request_t* req, + DataCompressor::Type response_compression_type, + const std::shared_ptr& triton_request, + const std::shared_ptr& shm_manager, + std::shared_ptr aggregator, const size_t slot) + : HTTPAPIServer::InferRequestClass(server, req, response_compression_type, triton_request, shm_manager, false /* pause */, false /* fini hook */), aggregator_(std::move(aggregator)), slot_(slot){} + + static void InferResponseComplete(TRITONSERVER_InferenceResponse* response, const uint32_t flags, void* userp) { + auto* infer_request = reinterpret_cast(userp); + + if (response != nullptr) { + ++infer_request->response_count_; + } + + TRITONSERVER_Error* err = nullptr; + evbuffer* shard_json = evbuffer_new(); + if (infer_request->response_count_ != 1) { + const std::string msg = std::string("expected a single response, got ") + std::to_string(infer_request->response_count_); + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, msg.c_str()); + } else if (response != nullptr) { + err = infer_request->FinalizeResponse(response, shard_json); +#ifdef TRITON_ENABLE_TRACING + if (infer_request->trace_ != nullptr) { + infer_request->trace_->CaptureTimestamp("INFER_RESPONSE_COMPLETE", TraceManager::CaptureTimestamp()); + } +#endif // TRITON_ENABLE_TRACING + } + + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceResponseDelete(response), "deleting inference response"); + + std::string json_fragment; + if (err == nullptr) { + const size_t len = evbuffer_get_length(shard_json); + if (len > 0) { + const unsigned char* p = evbuffer_pullup(shard_json, -1); + if (p != nullptr) { + json_fragment.assign(reinterpret_cast(p), len); + } + } + } + evbuffer_free(shard_json); + + if (err != nullptr) { + infer_request->aggregator_->OnShardDone(infer_request->slot_, err, ""); + } + else { + infer_request->aggregator_->OnShardDone(infer_request->slot_, nullptr, json_fragment); + } + + if ((flags & TRITONSERVER_RESPONSE_COMPLETE_FINAL) == 0) { + return; + } + evthr_defer(infer_request->thread_, DeleteMultiInferShardRequestThunk, infer_request); + } + + private: + static void DeleteMultiInferShardRequestThunk(evthr_t* /*thr*/, void* arg, void* /*shared*/) { + delete reinterpret_cast(arg); + } + + std::shared_ptr aggregator_; + const size_t slot_; +}; + +} // namespace + +void HTTPAPIServer::HandleMultiInfer(evhtp_request_t* req) { + RETURN_AND_RESPOND_IF_RESTRICTED(req, RestrictedCategory::INFERENCE, restricted_apis_); + + if (req->method != htp_method_POST) { + RETURN_AND_RESPOND_WITH_ERR(req, EVHTP_RES_METHNALLOWED, "Method Not Allowed"); + } + + evhtp_request_pause(req); + + evbuffer* decompressed_buffer = nullptr; + TRITONSERVER_Error* derr = DecompressBuffer(req, &decompressed_buffer); + if (derr != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, derr); + evhtp_send_reply(req, HttpCodeFromError(derr)); + TRITONSERVER_ErrorDelete(derr); + evhtp_request_resume(req); + return; + } + + evbuffer* body_buf = (decompressed_buffer != nullptr) ? decompressed_buffer : req->buffer_in; + + triton::common::TritonJson::Value root; + TRITONSERVER_Error* err = nullptr; + const size_t body_len = evbuffer_get_length(body_buf); + std::vector body_copy(body_len); + if (body_len > 0) { + const ssize_t nread = evbuffer_copyout(body_buf, body_copy.data(), body_len); + if (nread < 0 || static_cast(nread) != body_len) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "failed to read multi_infer request body"); + } + } + if (err == nullptr) { + const std::string body_json(body_copy.data(), body_len); + rapidjson::Document parsed; + err = triton::server::ParseRequest(body_json, server_.get(), &parsed); + if (err == nullptr) { + rapidjson::StringBuffer sb; + rapidjson::Writer writer(sb); + parsed.Accept(writer); + const std::string transformed(sb.GetString(), sb.GetSize()); + err = root.Parse(transformed.c_str(), transformed.size()); + } + } + if (decompressed_buffer != nullptr) { + evbuffer_free(decompressed_buffer); + decompressed_buffer = nullptr; + } + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + triton::common::TritonJson::Value requests; + if (!root.Find("requests", &requests)) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "Request body must include a JSON array field 'requests'"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + const size_t n = requests.ArraySize(); + if (n == 0) { + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "'requests' array must be non-empty"); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + if (n > kMaxMultiInferRequests) { + const std::string lim = "At most " + std::to_string(kMaxMultiInferRequests) + " sub-requests are allowed per multi_infer call"; + err = TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, lim.c_str()); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + struct SlotPrep { + std::string model_name; + int64_t model_version{0}; + std::string infer_body_json; + }; + std::vector slots; + slots.reserve(n); + + for (size_t i = 0; i < n; ++i) { + triton::common::TritonJson::Value slot; + err = requests.At(i, &slot); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + const char* mn_c; + size_t mn_len; + err = slot.MemberAsString("model_name", &mn_c, &mn_len); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + SlotPrep prep; + prep.model_name.assign(mn_c, mn_len); + std::string ver_str; + err = GetModelVersionStringFromSlot(slot, &ver_str); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = GetModelVersionFromString(ver_str, &prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + err = CheckTransactionPolicy(req, prep.model_name, prep.model_version); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + triton::common::TritonJson::Value infer_only; + err = CopyInferSlotBodyJson(slot, &infer_only); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + triton::common::TritonJson::WriteBuffer wb; + err = infer_only.Write(&wb); + if (err != nullptr) { + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + prep.infer_body_json.assign(wb.Base(), wb.Size()); + slots.push_back(std::move(prep)); + } + + evthr_t* reply_thread = evhtp_request_get_connection(req)->thread; + std::vector> irequests; + irequests.reserve(n); + for (size_t i = 0; i < n; ++i) { + TRITONSERVER_InferenceRequest* ireq = nullptr; + err = TRITONSERVER_InferenceRequestNew(&ireq, server_.get(), slots[i].model_name.c_str(), slots[i].model_version); + if (err != nullptr) { + for (auto& ir : irequests) { + if (ir != nullptr) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(ir.get()), "deleting unused multi_infer sub-request"); + } + } + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + irequests.emplace_back(ireq, [](TRITONSERVER_InferenceRequest* r) { + LOG_TRITONSERVER_ERROR(TRITONSERVER_InferenceRequestDelete(r),"deleting HTTP multi_infer sub-request"); + }); + } + + std::shared_ptr aggregator = std::make_shared(req, n, reply_thread, irequests); + std::vector> shard_holders; + std::vector> release_holders; + shard_holders.reserve(n); + release_holders.reserve(n); + + for (size_t i = 0; i < n; ++i) { + evbuffer* body_i = evbuffer_new(); + evbuffer_add(body_i, slots[i].infer_body_json.data(), slots[i].infer_body_json.size()); + const int32_t content_length = static_cast(evbuffer_get_length(body_i)); + size_t header_length = 0; + err = GetInferenceHeaderLength(req, content_length, &header_length); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto shard = std::make_unique(server_.get(), req, GetResponseCompressionType(req), irequests[i], shm_manager_, aggregator, i); + + err = EVRequestToTritonRequest(req, slots[i].model_name, irequests[i].get(), body_i, shard.get(), header_length); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + evbuffer_free(body_i); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + auto rel = std::make_unique(irequests[i], body_i); + err = ScheduleInferAsync(req, irequests[i].get(), shard.get(), rel.get(), nullptr, MultiInferShardRequest::InferResponseComplete); + if (err != nullptr) { + aggregator->CancelAllSubRequests(); + AddContentTypeHeader(req, "application/json"); + EVBufferAddErrorJson(req->buffer_out, err); + evhtp_send_reply(req, HttpCodeFromError(err)); + TRITONSERVER_ErrorDelete(err); + evhtp_request_resume(req); + return; + } + + shard_holders.push_back(std::move(shard)); + release_holders.push_back(std::move(rel)); + } + + for (size_t i = 0; i < n; ++i) { + release_holders[i].release(); + shard_holders[i].release(); + } +} + +}} // namespace triton::server diff --git a/src/mysql_odbc_connection_pool.cc b/src/mysql_odbc_connection_pool.cc new file mode 100644 index 0000000000..a563dc7629 --- /dev/null +++ b/src/mysql_odbc_connection_pool.cc @@ -0,0 +1,697 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "mysql_odbc_connection_pool.h" + +#ifdef TRITON_ENABLE_LOGGING +#include "triton/common/logging.h" +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace triton { namespace server { + +namespace { + +std::atomic g_global_mysql_odbc_pool{nullptr}; + +std::string BuildMySqlDriverConnectString(const DatabaseConfig& c) +{ + std::string driver = c.odbc_driver_name; + if (driver.empty()) { + driver = "MySQL ODBC 9.7 Unicode Driver"; + } else if (driver == "MySQL ODBC 8.0 Unicode Driver") { + driver = "MySQL ODBC 9.7 Unicode Driver"; + } + std::ostringstream conn; + conn << "DRIVER={" << driver << "};" << "SERVER=" << c.database_ip << ";" << "PORT=" << c.database_port << ";" << "UID={" << c.dsn_user_name << "};" << "PWD={" << c.dsn_user_password << "};"; + return conn.str(); +} + +} // namespace + +PooledOdbcConnection::PooledOdbcConnection() = default; +PooledOdbcConnection::PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc) : pool_(pool), dbc_(dbc){} +PooledOdbcConnection::~PooledOdbcConnection() +{ + Release(); +} + +PooledOdbcConnection::PooledOdbcConnection(PooledOdbcConnection&& other) noexcept : pool_(other.pool_), dbc_(other.dbc_) +{ + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; +} + +PooledOdbcConnection& PooledOdbcConnection::operator=(PooledOdbcConnection&& other) noexcept +{ + if (this != &other) { + Release(); + pool_ = other.pool_; + dbc_ = other.dbc_; + other.pool_ = nullptr; + other.dbc_ = SQL_NULL_HDBC; + } + return *this; +} + +void PooledOdbcConnection::Release() +{ + if (pool_ != nullptr && dbc_ != SQL_NULL_HDBC) { + pool_->ReturnConnection(dbc_); + } + pool_ = nullptr; + dbc_ = SQL_NULL_HDBC; +} + +MysqlOdbcConnectionPool::MysqlOdbcConnectionPool(DatabaseConfig config) : config_(std::move(config)){} +MysqlOdbcConnectionPool::~MysqlOdbcConnectionPool() +{ + std::lock_guard lk(mu_); + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } +} + +std::optional MysqlOdbcConnectionPool::Initialize() +{ + auto cleanup_partial = [this]() { + for (SQLHDBC dbc : all_handles_) { + if (dbc != SQL_NULL_HDBC) { + SQLDisconnect(dbc); + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + } + } + all_handles_.clear(); + free_.clear(); + if (henv_ != SQL_NULL_HENV) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + } + }; + + SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &henv_); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLAllocHandle(ENV) failed"); + } + + rc = SQLSetEnvAttr(henv_, SQL_ATTR_ODBC_VERSION, (SQLPOINTER)SQL_OV_ODBC3, 0); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_ENV, henv_); + henv_ = SQL_NULL_HENV; + return std::string("SQLSetEnvAttr failed"); + } + + const std::size_t pool_size = config_.max_pool_connections; + all_handles_.reserve(pool_size); + + for (std::size_t i = 0; i < pool_size; ++i) { + SQLHDBC dbc = SQL_NULL_HDBC; + rc = SQLAllocHandle(SQL_HANDLE_DBC, henv_, &dbc); + if (!SQL_SUCCEEDED(rc)) { + cleanup_partial(); + return std::string("SQLAllocHandle(DBC) failed"); + } + + if (!config_.database_ip.empty()) { + const std::string conn_str = BuildMySqlDriverConnectString(config_); + SQLCHAR out_conn[1024]{}; + SQLSMALLINT out_conn_len = 0; + rc = SQLDriverConnect(dbc, nullptr, reinterpret_cast(const_cast(conn_str.data())), + SQL_NTS, out_conn, sizeof(out_conn), &out_conn_len, SQL_DRIVER_NOPROMPT); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLDriverConnect failed"); + } + } else { + rc = SQLConnect(dbc,reinterpret_cast(const_cast(config_.primary_dsn_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_name.data())), + SQL_NTS, reinterpret_cast(const_cast(config_.dsn_user_password.data())), + SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + SQLFreeHandle(SQL_HANDLE_DBC, dbc); + cleanup_partial(); + return std::string("SQLConnect failed"); + } + } + + all_handles_.push_back(dbc); + free_.push_back(dbc); + } + + return std::nullopt; +} + +PooledOdbcConnection MysqlOdbcConnectionPool::Acquire() +{ + std::unique_lock lk(mu_); + cv_.wait(lk, [this] { return !free_.empty(); }); + SQLHDBC dbc = free_.front(); + free_.pop_front(); + return PooledOdbcConnection(this, dbc); +} + +void MysqlOdbcConnectionPool::ReturnConnection(SQLHDBC dbc) +{ + { + std::lock_guard lk(mu_); + free_.push_back(dbc); + } + cv_.notify_one(); +} + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool) +{ + g_global_mysql_odbc_pool.store(pool, std::memory_order_release); +} + +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool() +{ + return g_global_mysql_odbc_pool.load(std::memory_order_acquire); +} + +namespace { + + +constexpr const char kSqlBtModelsMaxTs[] = "SELECT UNIX_TIMESTAMP(MAX(update_timestamp)) FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1"; +constexpr const char kSqlBtModelsForDc[] = "SELECT campaign_id, LOWER(model_name), feature_mapping, feature_sequence, applicable_campaigns FROM MLBasedThrottling.lightgbm_bt_models WHERE on_off = 1 AND dc_id = ?"; +constexpr size_t kModelNameBuf = kMaxModelNameLen + 1; +constexpr size_t kFeatureMappingBuf = kMaxFeatureMappingJsonLen + 1; +constexpr size_t kFeatureSequenceBuf = kMaxFeatureSequenceLen + 1; +constexpr size_t kApplicableCampaignsBuf = kMaxApplicableCampaignsLen + 1; + +struct OdbcStmt { + SQLHSTMT h{SQL_NULL_HSTMT}; + explicit OdbcStmt(SQLHDBC dbc) + { + const SQLRETURN rc = SQLAllocHandle(SQL_HANDLE_STMT, dbc, &h); + if (!SQL_SUCCEEDED(rc)) { + h = SQL_NULL_HSTMT; + } + } + ~OdbcStmt() + { + if (h != SQL_NULL_HSTMT) { + SQLFreeHandle(SQL_HANDLE_STMT, h); + } + } + OdbcStmt(const OdbcStmt&) = delete; + OdbcStmt& operator=(const OdbcStmt&) = delete; +}; + +void Trim(std::string* s) +{ + if (s == nullptr || s->empty()) { + return; + } + const auto not_space = [](unsigned char c) { return !std::isspace(c); }; + auto b = std::find_if(s->begin(), s->end(), not_space); + auto e = std::find_if(s->rbegin(), s->rend(), not_space).base(); + if (b >= e) { + s->clear(); + } else { + *s = std::string(b, e); + } +} + +std::string SqlCharBufferToString(const std::vector& buf, SQLLEN cb) +{ + if (cb == SQL_NULL_DATA) { + return {}; + } + const char* p = reinterpret_cast(buf.data()); + if (cb < 0) { + return std::string(p); + } + return std::string(p, static_cast(cb)); +} + +std::string JsonScalarToString(const rapidjson::Value& v) +{ + if (v.IsString()) { + return std::string(v.GetString(), v.GetStringLength()); + } + if (v.IsBool()) { + return v.GetBool() ? "true" : "false"; + } + if (v.IsInt()) { + return std::to_string(v.GetInt()); + } + if (v.IsUint()) { + return std::to_string(v.GetUint()); + } + if (v.IsInt64()) { + return std::to_string(v.GetInt64()); + } + if (v.IsUint64()) { + return std::to_string(v.GetUint64()); + } + if (v.IsDouble()) { + return std::to_string(v.GetDouble()); + } + if (v.IsNull()) { + return {}; + } + return {}; +} + +std::optional FetchMaxUnixTimestampFromDbc(SQLHDBC dbc, const char* sql, int64_t* out_ts) +{ + *out_ts = 0; + OdbcStmt st(dbc); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + SQLRETURN rc = SQLExecDirect(st.h, reinterpret_cast(const_cast(sql)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecDirect failed"); + } + int64_t ts = 0; + SQLLEN cb_ts = 0; + rc = SQLBindCol(st.h, 1, SQL_C_SBIGINT, &ts, 0, &cb_ts); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol failed"); + } + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + return std::nullopt; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_ts != SQL_NULL_DATA) { + *out_ts = ts; + } + return std::nullopt; +} + +} // namespace + +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out) +{ + out->clear(); + std::size_t start = 0; + while (start < s.size()) { + const std::size_t comma = s.find(',', start); + std::string piece = (comma == std::string::npos) ? s.substr(start) : s.substr(start, comma - start); + Trim(&piece); + if (!piece.empty()) { + out->push_back(std::move(piece)); + } + if (comma == std::string::npos) { + break; + } + start = comma + 1; + } +} + +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out) +{ + out->clear(); + std::string t = s; + Trim(&t); + if (t.empty()) { + return std::nullopt; + } + if (t.size() == 2 && (t[0] == 'n' || t[0] == 'N') && (t[1] == 'a' || t[1] == 'A')) { + return std::nullopt; + } + + std::vector tokens; + SplitCommaSeparatedStrings(t, &tokens); + if (tokens.empty()) { + return std::nullopt; + } + + for (const auto& tok : tokens) { + char* endptr = nullptr; + const long v = std::strtol(tok.c_str(), &endptr, 10); + if (endptr == tok.c_str() || *endptr != '\0') { + return std::string("invalid campaign id token: ") + tok; + } + out->push_back(static_cast(v)); + } + return std::nullopt; +} + +int GetFeatureMappingIdx(const char* feature_name, const char* feature, const FeatureMappingTables* feature_mapping) +{ + if (feature_mapping == nullptr || feature_name == nullptr || feature == nullptr) { + return -1; + } + const auto outer = feature_mapping->find(feature_name); + if (outer == feature_mapping->end()) { + return -1; + } + const auto inner = outer->second.value_to_index.find(feature); + if (inner == outer->second.value_to_index.end()) { + return -1; + } + return inner->second; +} + +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error) +{ + out->clear(); + if (parse_error != nullptr) { + parse_error->clear(); + } + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + if (parse_error != nullptr) { + *parse_error = std::string("JSON parse error at offset ") + std::to_string(doc.GetErrorOffset()) + ": " + rapidjson::GetParseError_En(doc.GetParseError()); + } + return false; + } + if (!doc.IsObject()) { + if (parse_error != nullptr) { + *parse_error = "feature_mapping root must be a JSON object"; + } + return false; + } + + for (auto it = doc.MemberBegin(); it != doc.MemberEnd(); ++it) { + if (!it->name.IsString()) { + continue; + } + const std::string feature_name(it->name.GetString(), it->name.GetStringLength()); + if (feature_name.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "feature name exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1"; + } + return false; + } + if (!it->value.IsArray()) { + if (parse_error != nullptr) { + *parse_error = "feature '" + feature_name + "' value is not a JSON array"; + } + return false; + } + const rapidjson::Value& arr = it->value; + FeatureValueIndexMap table; + table.values.reserve(arr.Size()); + for (rapidjson::SizeType i = 0; i < arr.Size(); ++i) { + const std::string cell = JsonScalarToString(arr[i]); + if (cell.size() > kTritonFeatureMappingMaxTokenLen) { + if (parse_error != nullptr) { + *parse_error = "categorical value exceeds TRITON_FEATURE_MAPPING_BUFF_SIZE-1 for feature '" + feature_name + "'"; + } + return false; + } + table.values.push_back(cell); + table.value_to_index[cell] = static_cast(i); + } + (*out)[feature_name] = std::move(table); + } + return true; +} + +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts) +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + return FetchMaxUnixTimestampFromDbc(conn.handle(), kSqlBtModelsMaxTs, out_ts); +} + +namespace { + +bool TryMergeRowIntoCampaignMap(const LightgbmBtModelRow& row, CampaignToFeatureMappings& out) +{ + const CampaignBtModelBundle bundle{row.model_name, row.feature_mapping, row.feature_sequence}; + + if (row.campaign_id != 0) { + if (out.find(row.campaign_id) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << row.campaign_id << " already exists in triton models map; skipping row"; +#endif + return false; + } + out[row.campaign_id] = bundle; + return true; + } + + if (row.applicable_campaigns.empty()) { + if (out.find(0) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign 0 already exists in triton models map; skipping row"; +#endif + return false; + } + out[0] = bundle; + return true; + } + + bool any_inserted = false; + for (int32_t cid : row.applicable_campaigns) { + if (out.find(cid) != out.end()) { +#ifdef TRITON_ENABLE_LOGGING + LOG_WARNING << "TRITON: campaign " << cid << " already exists in triton models map"; +#endif + continue; + } + out[cid] = bundle; + any_inserted = true; + } + return any_inserted; +} + +} // namespace + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map) +{ + out_campaign_map.clear(); + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return std::string("MySQL ODBC pool is not registered; call SetGlobalMysqlOdbcPool after pool init"); + } + const int32_t dc_id = pool->Config().dc_id; + PooledOdbcConnection conn = pool->Acquire(); + if (!conn) { + return std::string("failed to acquire ODBC connection"); + } + + OdbcStmt st(conn.handle()); + if (st.h == SQL_NULL_HSTMT) { + return std::string("SQLAllocHandle(STMT) failed"); + } + + SQLRETURN rc = SQLPrepare(st.h, reinterpret_cast(const_cast(kSqlBtModelsForDc)), SQL_NTS); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLPrepare failed"); + } + + SQLINTEGER dc_param = static_cast(dc_id); + rc = SQLBindParameter(st.h, 1, SQL_PARAM_INPUT, SQL_C_SLONG, SQL_INTEGER, 0, 0, &dc_param, 0, nullptr); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindParameter failed"); + } + + rc = SQLExecute(st.h); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLExecute failed"); + } + + SQLINTEGER s_campaign_id = 0; + SQLLEN cb_campaign_id = 0; + std::vector model_buf(kModelNameBuf + 1, 0); + SQLLEN cb_model_name = 0; + std::vector mapping_buf(kFeatureMappingBuf + 1, 0); + SQLLEN cb_mapping = 0; + std::vector sequence_buf(kFeatureSequenceBuf + 1, 0); + SQLLEN cb_sequence = 0; + std::vector campaigns_buf(kApplicableCampaignsBuf + 1, 0); + SQLLEN cb_campaigns = 0; + + int col = 1; + rc = SQLBindCol(st.h, col++, SQL_C_SLONG, &s_campaign_id, 0, &cb_campaign_id); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(campaign_id) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, model_buf.data(), static_cast(model_buf.size()), &cb_model_name); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(model_name) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, mapping_buf.data(), static_cast(mapping_buf.size()), &cb_mapping); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_mapping) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, sequence_buf.data(), static_cast(sequence_buf.size()), &cb_sequence); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(feature_sequence) failed"); + } + rc = SQLBindCol(st.h, col++, SQL_C_CHAR, campaigns_buf.data(), static_cast(campaigns_buf.size()), &cb_campaigns); + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLBindCol(applicable_campaigns) failed"); + } + + while (true) { + rc = SQLFetch(st.h); + if (rc == SQL_NO_DATA) { + break; + } + if (!SQL_SUCCEEDED(rc)) { + return std::string("SQLFetch failed"); + } + if (cb_campaign_id == SQL_NULL_DATA) { + continue; + } + + LightgbmBtModelRow row; + row.campaign_id = static_cast(s_campaign_id); + + if (cb_model_name != SQL_NULL_DATA) { + row.model_name = SqlCharBufferToString(model_buf, cb_model_name); + Trim(&row.model_name); + } + + std::string mapping_json; + if (cb_mapping != SQL_NULL_DATA) { + mapping_json = SqlCharBufferToString(mapping_buf, cb_mapping); + Trim(&mapping_json); + } + std::string parse_err; + if (!ParseFeatureMappingJson(mapping_json, &row.feature_mapping, &parse_err)) { + return std::string("feature_mapping JSON: ") + parse_err; + } + + std::string seq_str; + if (cb_sequence != SQL_NULL_DATA) { + seq_str = SqlCharBufferToString(sequence_buf, cb_sequence); + Trim(&seq_str); + } + SplitCommaSeparatedStrings(seq_str, &row.feature_sequence); + + std::string camp_str; + if (cb_campaigns != SQL_NULL_DATA) { + camp_str = SqlCharBufferToString(campaigns_buf, cb_campaigns); + Trim(&camp_str); + } + if (auto err = ParseApplicableCampaignIds(camp_str, &row.applicable_campaigns)) { + return err; + } + + if (!TryMergeRowIntoCampaignMap(row, out_campaign_map)) { + continue; + } + } + + return std::nullopt; +} + +namespace { + std::array g_triton_campaign_feature_mappings; + std::atomic g_triton_models_active{0}; + std::atomic g_triton_models_modification_time{0}; +} + +bool IsTritonModelsModified() +{ + MysqlOdbcConnectionPool* pool = GlobalMysqlOdbcPool(); + if (pool == nullptr) { + return false; + } + + int retry_count = pool->Config().query_retry_count; + if (retry_count < 0) { + retry_count = 0; + } + + int64_t last_updated = 0; + std::optional err; + do { + err = FetchLightgbmBtModelsMaxUpdateUnixSeconds(&last_updated); + if (!err) { + break; + } + } while (err.has_value() && retry_count--); + + if (err.has_value()) { + return false; + } + + const uint32_t lu = static_cast(last_updated); + const uint32_t prev = g_triton_models_modification_time.load(std::memory_order_relaxed); + if (lu > prev) { + g_triton_models_modification_time.store(lu, std::memory_order_relaxed); + return true; + } + return false; +} + +std::optional UpdateTritonModelsData() +{ + if (!IsTritonModelsModified()) { + return std::nullopt; + } + + CampaignToFeatureMappings by_campaign; + if (auto err = FetchLightgbmBtModelsForDc(by_campaign)) { + return err; + } + + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + const int inactive = 1 - idx; + g_triton_campaign_feature_mappings[inactive] = std::move(by_campaign); + g_triton_models_active.store(inactive, std::memory_order_release); + g_triton_campaign_feature_mappings[idx].clear(); + return std::nullopt; +} + +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings() +{ + const int idx = g_triton_models_active.load(std::memory_order_acquire) & 1; + return &g_triton_campaign_feature_mappings[idx]; +} + +}} // namespace triton::server \ No newline at end of file diff --git a/src/mysql_odbc_connection_pool.h b/src/mysql_odbc_connection_pool.h new file mode 100644 index 0000000000..771fcefbfb --- /dev/null +++ b/src/mysql_odbc_connection_pool.h @@ -0,0 +1,163 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Connection pool over the MySQL ODBC driver using the ODBC API (unixODBC / +// iODBC). Build with -DTRITON_ENABLE_MYSQL_ODBC=ON and install unixodbc-dev +// (Debian/Ubuntu) plus a configured MySQL ODBC DSN. + +#pragma once + +#include "database_config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +#include +#include + +namespace triton { namespace server { + +inline constexpr std::size_t kMaxModelNameLen = 100; +inline constexpr std::size_t kMaxFeatureMappingJsonLen = 16 * (1 << 20); +inline constexpr std::size_t kMaxFeatureSequenceLen = 1000; +inline constexpr std::size_t kMaxApplicableCampaignsLen = 4096; +inline constexpr std::size_t kMaxLightgbmUpdateTimestampLen = 256; +inline constexpr std::size_t kTritonFeatureMappingBuffSize = 256; +inline constexpr std::size_t kTritonFeatureMappingMaxTokenLen = + kTritonFeatureMappingBuffSize - 1; + +class MysqlOdbcConnectionPool; + +class PooledOdbcConnection { + public: + PooledOdbcConnection(); + ~PooledOdbcConnection(); + + PooledOdbcConnection(PooledOdbcConnection&& other) noexcept; + PooledOdbcConnection& operator=(PooledOdbcConnection&& other) noexcept; + + PooledOdbcConnection(const PooledOdbcConnection&) = delete; + PooledOdbcConnection& operator=(const PooledOdbcConnection&) = delete; + + SQLHDBC handle() const { return dbc_; } + explicit operator bool() const { return dbc_ != SQL_NULL_HDBC; } + + private: + friend class MysqlOdbcConnectionPool; + PooledOdbcConnection(MysqlOdbcConnectionPool* pool, SQLHDBC dbc); + + void Release(); + + MysqlOdbcConnectionPool* pool_{nullptr}; + SQLHDBC dbc_{SQL_NULL_HDBC}; +}; + +class MysqlOdbcConnectionPool { + public: + explicit MysqlOdbcConnectionPool(DatabaseConfig config); + ~MysqlOdbcConnectionPool(); + + MysqlOdbcConnectionPool(const MysqlOdbcConnectionPool&) = delete; + MysqlOdbcConnectionPool& operator=(const MysqlOdbcConnectionPool&) = delete; + + std::optional Initialize(); + + PooledOdbcConnection Acquire(); + + const DatabaseConfig& Config() const { return config_; } + + private: + friend class PooledOdbcConnection; + void ReturnConnection(SQLHDBC dbc); + + DatabaseConfig config_; + SQLHENV henv_{SQL_NULL_HENV}; + std::vector all_handles_; + std::deque free_; + std::mutex mu_; + std::condition_variable cv_; +}; + +void SetGlobalMysqlOdbcPool(MysqlOdbcConnectionPool* pool); +MysqlOdbcConnectionPool* GlobalMysqlOdbcPool(); + +struct FeatureValueIndexMap { + std::vector values; + std::unordered_map value_to_index; +}; + +using FeatureMappingTables = std::unordered_map; + +// Legacy get_feature_mapping_idx: look up categorical index for `feature` +// (token string) under column `feature_name`. Returns -1 if any map or key is +// missing, or if `feature_name` / `feature` / `feature_mapping` is null. +int GetFeatureMappingIdx( + const char* feature_name, const char* feature, + const FeatureMappingTables* feature_mapping); + +// Loaded from `lightgbm_bt_models` per campaign_id (after merge rules). +struct CampaignBtModelBundle { + std::string model_name; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; +}; + +using CampaignToFeatureMappings = std::unordered_map; + +struct LightgbmBtModelRow { + int32_t campaign_id{0}; + std::string model_name; + FeatureMappingTables feature_mapping; + std::vector feature_sequence; + std::vector applicable_campaigns; +}; + +std::optional FetchLightgbmFeatureMappingMaxUpdateUnixSeconds(int64_t* out_ts); +std::optional FetchLightgbmBtModelsMaxUpdateUnixSeconds(int64_t* out_ts); + +std::optional FetchLightgbmBtModelsForDc(CampaignToFeatureMappings& out_campaign_map); +bool ParseFeatureMappingJson(const std::string& json, FeatureMappingTables* out, std::string* parse_error); +void SplitCommaSeparatedStrings(const std::string& s, std::vector* out_tokens); +std::optional ParseApplicableCampaignIds(const std::string& s, std::vector* out_ids); +bool IsTritonModelsModified(); +std::optional UpdateTritonModelsData(); +const CampaignToFeatureMappings* ActiveCampaignToFeatureMappings(); + +}} // namespace triton::server + diff --git a/src/orca_http.cc b/src/orca_http.cc new file mode 100644 index 0000000000..082a17180e --- /dev/null +++ b/src/orca_http.cc @@ -0,0 +1,233 @@ +// Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "orca_http.h" + +void +SetEndpointLoadMetricsHeader( + evhtp_request_t* req, const char* orca_metric_format, + TRITONSERVER_Server* server) +{ + const std::string orca_type = orca_metric_format; + TRITONSERVER_Metrics* metrics = nullptr; + TRITONSERVER_Error* err = TRITONSERVER_ServerMetrics(server, &metrics); + if (err == nullptr) { + const char* base; + size_t byte_size; + err = TRITONSERVER_MetricsFormatted( + metrics, TRITONSERVER_METRIC_PROMETHEUS, &base, &byte_size); + if (err == nullptr) { + std::string formatted_metrics(base, byte_size); + // Extract the KV utilization metrics from the Prometheus formatted + // string. + std::string extracted_kv_metrics = + ExtractKVMetrics(formatted_metrics, orca_type); + if (!extracted_kv_metrics.empty()) { + evhtp_headers_add_header( + req->headers_out, evhtp_header_new( + ENDPOINT_LOAD_METRICS_NAME, + extracted_kv_metrics.c_str(), 1, 1)); + } else { + LOG_ERROR << "ENDPOINT_LOAD_METRICS_TYPE request header is set but " + "extracted_kv_metrics is " + "empty, no header written. orca_type=" + << orca_type; + } + } + } else { + // Handle potential errors + LOG_ERROR << "Failed to get KV metrics: " << TRITONSERVER_ErrorMessage(err); + TRITONSERVER_ErrorDelete(err); + } + TRITONSERVER_MetricsDelete(metrics); +} + +std::vector +MetricFamilyExtractor(const std::string& input, const std::string& metricFamily) +{ + std::vector metrics; + // Construct the regex pattern using the provided metricFamily. + + // `labelGroup` is a capturing group that captures all characters within curly + // braces, excluding line breaks. + std::string labelGroup = "(?:{(.*?)})"; + + // `valueGroup` is a capturing group that captures a number with its + // decimals if any. + std::string valueGroup = R"((\d+(?:\.\d+)?))"; + + // `patternStr` matches on lines starting with `metricFamily` then captures + // its labels if any, then (optionally) matches any whitespace, then captures + // its numeric double value. + // + // For example, `patternStr` would match on input: + // `nv_trt_llm_kv_cache_block_metrics{kv_cache_block_type="used",model="tensorrt_llm",version="1"} + // 3` + // + // with 2 capturing groups: + // 1. `kv_cache_block_type="used",model="tensorrt_llm",version="1"` + // 2. `3` + std::string patternStr = metricFamily + labelGroup + R"(?\s*)" + valueGroup; + re2::RE2 pattern(patternStr); + re2::StringPiece inputPiece(input); + + std::string labelString; + std::string metric_value; + + while (re2::RE2::FindAndConsume( + &inputPiece, pattern, &labelString, &metric_value)) { + PromMetric metric; + + // Extract labels if they exist + if (!labelString.empty()) { + // `labelPattern` captures any alphanumeric sequence that precedes an '=' + // character, then captures the following quoted character sequence. These + // groups are exahstive given the prometheus data model: + // https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels + // + // For example, calling FindAndConsume() with `labelPattern` on input: + // `kv_cache_block_type="used",model="tensorrt_llm",version="1"` + // + // matches 3 times with 2 capturing groups each: + // + // Match #1 + // 1. `kv_cache_block_type` + // 2. `used` + // + // Match #2 + // 1. `model` + // 2. `tensorrt_llm` + // + // Match #3 + // 1. `version` + // 2. `1` + re2::RE2 labelPattern(R"((\w+)=\"([^\"]*)\")"); + re2::StringPiece labelPiece(labelString); + std::string key, value; + while ( + re2::RE2::FindAndConsume(&labelPiece, labelPattern, &key, &value)) { + // Populate the metric's labels map + metric.labels[key] = value; + } + } + + // Assign the metric its value and add it to the family list + metric.value = stod(metric_value); + metrics.push_back(metric); + } + + return metrics; +} + +std::string +ExtractKVMetrics( + const std::string& prometheus_metrics, const std::string& orca_type) +{ + std::string metric_family = KV_CACHE_BLOCK_METRICS_FAMILY; + std::vector kv_cache_metrics = + MetricFamilyExtractor(prometheus_metrics, metric_family); + + double tokens_per_block = -1; + double used_blocks = -1; + double max_blocks = -1; + + for (const auto& metric : kv_cache_metrics) { + if (metric.labels.count(KV_CACHE_BLOCK_TYPE) > 0) { + std::string type = metric.labels.at(KV_CACHE_BLOCK_TYPE); + if (type == KV_CACHE_BLOCK_TYPE_TOKENS_PER) { + tokens_per_block = metric.value; + } else if (type == KV_CACHE_BLOCK_TYPE_USED) { + used_blocks = metric.value; + } else if (type == KV_CACHE_BLOCK_TYPE_MAX) { + max_blocks = metric.value; + } + } + } + + // Return early if not all kv metrics are found and set. + if (tokens_per_block < 0 || used_blocks < 0 || max_blocks < 0) { + LOG_ERROR << "One or more of the kv metrics was not found or invalid."; + return ""; + } + + // Calculate derived metrics + double kv_cache_utilization = 0; + if (max_blocks > 0) { + kv_cache_utilization = used_blocks / max_blocks; + } + double max_token_capacity = max_blocks * tokens_per_block; + + std::unordered_map + metrics; // metrics vector to pass down + metrics[KV_CACHE_UTIL_KEY] = kv_cache_utilization; + metrics[MAX_TOKEN_CAPACITY_KEY] = max_token_capacity; + + return OrcaKVMetricHeader(orca_type, metrics); +} + +std::string +OrcaKVMetricHeader( + const std::string& orca_type, + std::unordered_map metrics) +{ + // Logic to construct and format response header + std::string header_contents = ""; + const std::string named_metrics_key = NAMED_METRICS; + const std::string kv_util_key = KV_CACHE_UTIL_KEY; + const std::string max_token_key = MAX_TOKEN_CAPACITY_KEY; + + if (orca_type == "json") { + // Format the metrics according to the ORCA protocol as JSON. + triton::common::TritonJson::Value orca_metrics( + triton::common::TritonJson::ValueType::OBJECT); + triton::common::TritonJson::Value named_metrics( + orca_metrics, triton::common::TritonJson::ValueType::OBJECT); + + named_metrics.AddDouble(kv_util_key.c_str(), metrics[kv_util_key]); + named_metrics.AddUInt(max_token_key.c_str(), metrics[max_token_key]); + orca_metrics.Add(named_metrics_key.c_str(), std::move(named_metrics)); + + triton::common::TritonJson::WriteBuffer buffer; + orca_metrics.Write(&buffer); + header_contents = std::string("JSON ") + buffer.Contents(); + + } else if (orca_type == "text") { + // Format the metrics according to the ORCA protocol as Native HTTP + // (comma separated list). + const std::string prefix = named_metrics_key + "."; + + header_contents = "TEXT "; + header_contents += prefix + kv_util_key + "=" + + std::to_string(metrics[kv_util_key]) + ", "; + header_contents += + prefix + max_token_key + "=" + + std::to_string(static_cast(metrics[max_token_key])); + } else { + LOG_ERROR << "orca_type is set to an invalid type: " << orca_type; + } + + return header_contents; +} diff --git a/src/orca_http.h b/src/orca_http.h new file mode 100644 index 0000000000..ffb367bd4b --- /dev/null +++ b/src/orca_http.h @@ -0,0 +1,67 @@ +// Copyright 2020-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include + +#include "http_server.h" + +#define ENDPOINT_LOAD_METRICS_TYPE "endpoint-load-metrics-format" +#define ENDPOINT_LOAD_METRICS_NAME "endpoint-load-metrics" +#define KV_CACHE_BLOCK_METRICS_FAMILY "nv_trt_llm_kv_cache_block_metrics" +#define KV_CACHE_BLOCK_TYPE "kv_cache_block_type" +#define KV_CACHE_BLOCK_TYPE_TOKENS_PER "tokens_per" +#define KV_CACHE_BLOCK_TYPE_USED "used" +#define KV_CACHE_BLOCK_TYPE_MAX "max" +#define KV_CACHE_UTIL_KEY "kv_cache_utilization" +#define MAX_TOKEN_CAPACITY_KEY "max_token_capacity" +#define NAMED_METRICS "named_metrics" + +struct PromMetric { + std::unordered_map labels; + double value; +}; + +// function with logic to pull the KV-cache metrics for the inference +// response header +void SetEndpointLoadMetricsHeader( + evhtp_request_t* req, const char* orca_metric_format, + TRITONSERVER_Server* server); +// Helper function to get the KV-cache utilization metrics for the +// inference response header +std::string ExtractKVMetrics( + const std::string& prometheus_metrics, const std::string& orca_type); +// Generates a metric struct for a given family with a map of labels and a +// value +std::vector MetricFamilyExtractor( + const std::string& input, const std::string& metricFamily); +// Creates a header string in the the proper reporting format for provided +// KV-cache metrics. +std::string OrcaKVMetricHeader( + const std::string& reporting_format, + const std::unordered_map metrics); diff --git a/src/python/examples/example_model_repository/identity/1/model.onnx b/src/python/examples/example_model_repository/identity/1/model.onnx new file mode 100755 index 0000000000..eda3ec8d96 Binary files /dev/null and b/src/python/examples/example_model_repository/identity/1/model.onnx differ diff --git a/src/python/examples/example_model_repository/identity/1/model.savedmodel/saved_model.pb b/src/python/examples/example_model_repository/identity/1/model.savedmodel/saved_model.pb deleted file mode 100755 index 63f78fecb4..0000000000 Binary files a/src/python/examples/example_model_repository/identity/1/model.savedmodel/saved_model.pb and /dev/null differ diff --git a/src/python/examples/example_model_repository/identity/config.pbtxt b/src/python/examples/example_model_repository/identity/config.pbtxt index ae83e47556..0ea4e80c26 100644 --- a/src/python/examples/example_model_repository/identity/config.pbtxt +++ b/src/python/examples/example_model_repository/identity/config.pbtxt @@ -1,4 +1,5 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,14 +26,18 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. name: "identity" -platform: "tensorflow_savedmodel" +platform: "onnxruntime_onnx" max_batch_size: 8 +version_policy: { latest { num_versions: 1 }} + + input [ { name: "INPUT0" data_type: TYPE_STRING dims: [ -1 ] + } ] output [ @@ -40,5 +45,7 @@ output [ name: "OUTPUT0" data_type: TYPE_STRING dims: [ -1 ] + + } ] diff --git a/src/python/setup.py b/src/python/setup.py index ee1e7c0ec4..067ff61035 100755 --- a/src/python/setup.py +++ b/src/python/setup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -107,6 +107,6 @@ def get_tag(self): zip_safe=False, cmdclass={"bdist_wheel": bdist_wheel}, data_files=data_files, - install_requires=["tritonserver", "pydantic"], + install_requires=["tritonserver", "pydantic==2.10.6"], extras_require={"GPU": gpu_extras, "test": test_extras, "all": all_extras}, ) diff --git a/src/python/tritonfrontend/_api/_kservegrpc.py b/src/python/tritonfrontend/_api/_kservegrpc.py index efa706a77a..54959e1c89 100644 --- a/src/python/tritonfrontend/_api/_kservegrpc.py +++ b/src/python/tritonfrontend/_api/_kservegrpc.py @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -80,6 +80,7 @@ class Options: int, Grpc_compression_level ] = Grpc_compression_level.NONE infer_allocation_pool_size: int = Field(8, ge=0) + max_response_pool_size: int = Field(2_147_483_647, ge=0) forward_header_pattern: str = "" # DLIS-7215: Add restricted protocol support # restricted_protocols: str = "" diff --git a/src/python/tritonfrontend/_api/_kservegrpc.pyi b/src/python/tritonfrontend/_api/_kservegrpc.pyi index ae5d44ac96..0c8dc04804 100644 --- a/src/python/tritonfrontend/_api/_kservegrpc.pyi +++ b/src/python/tritonfrontend/_api/_kservegrpc.pyi @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -57,6 +57,7 @@ class KServeGrpc: max_connection_age_grace_ms: int infer_compression_level: int | Grpc_compression_level infer_allocation_pool_size: int + max_response_pool_size: int forward_header_pattern: str def __post_init__(self) -> None: ... triton_frontend: Incomplete diff --git a/src/sagemaker_server.h b/src/sagemaker_server.h index dcd40e66ac..ffb94322df 100644 --- a/src/sagemaker_server.h +++ b/src/sagemaker_server.h @@ -53,10 +53,12 @@ class SagemakerAPIServer : public HTTPAPIServer { TRITONSERVER_Server* server, evhtp_request_t* req, DataCompressor::Type response_compression_type, const std::shared_ptr& triton_request, - const std::shared_ptr& shm_manager) + const std::shared_ptr& shm_manager, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) : InferRequestClass( server, req, response_compression_type, triton_request, - shm_manager) + shm_manager, pause_http_request, register_fini_cancel_hook) { } using InferRequestClass::InferResponseComplete; @@ -123,12 +125,13 @@ class SagemakerAPIServer : public HTTPAPIServer { std::unique_ptr CreateInferRequest( evhtp_request_t* req, - const std::shared_ptr& triton_request) - override + const std::shared_ptr& triton_request, + bool pause_http_request = true, + bool register_fini_cancel_hook = true) override { return std::unique_ptr(new SagemakeInferRequestClass( server_.get(), req, GetResponseCompressionType(req), triton_request, - shm_manager_)); + shm_manager_, pause_http_request, register_fini_cancel_hook)); } TRITONSERVER_Error* GetInferenceHeaderLength( evhtp_request_t* req, int32_t content_length, diff --git a/src/transform.cc b/src/transform.cc new file mode 100644 index 0000000000..eb2f46e0f7 --- /dev/null +++ b/src/transform.cc @@ -0,0 +1,349 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifdef TRITON_ENABLE_MYSQL_ODBC +#include "transform.h" +#include "mysql_odbc_connection_pool.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int FeatureIdxFromJsonValue(const char* feature_name, const rapidjson::Value& v, const triton::server::FeatureMappingTables* tables) { + if (tables == nullptr) { + return -1; + } + if (v.IsString()) { + return triton::server::GetFeatureMappingIdx(feature_name, v.GetString(), tables); + } + char num_buf[48]; + int n = 0; + if (v.IsInt()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%d", v.GetInt()); + } else if (v.IsUint()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%u", v.GetUint()); + } else if (v.IsInt64()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%lld", static_cast(v.GetInt64())); + } else if (v.IsUint64()) { + n = std::snprintf(num_buf, sizeof(num_buf), "%llu", static_cast(v.GetUint64())); + } else { + return -1; + } + if (n <= 0 || static_cast(n) >= sizeof(num_buf)) { + return -1; + } + return triton::server::GetFeatureMappingIdx(feature_name, num_buf, tables); +} +} // namespace + +namespace triton { namespace server { + +TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { + if (out_doc == nullptr) { + return TRITONSERVER_ErrorNew( + TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); + } + + rapidjson::Document doc; + doc.Parse(json.data(), json.size()); + if (doc.HasParseError()) { + const char* msg = rapidjson::GetParseError_En(doc.GetParseError()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, msg); + } + + if (server != nullptr && doc.IsObject() && doc.HasMember("imps")) { + const rapidjson::Value& imps_member = doc["imps"]; + if (imps_member.IsArray()) { + return GenerateInputVectors(doc, server, out_doc); + } + } + + *out_doc = std::move(doc); + return nullptr; +} + +TRITONSERVER_Error* GetReadyModelNames(TRITONSERVER_Server* server, std::unordered_set* out) { + if (out == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output set pointer is null"); + } + out->clear(); + + TRITONSERVER_Message* message = nullptr; + TRITONSERVER_Error* err = TRITONSERVER_ServerModelIndex(server, TRITONSERVER_INDEX_FLAG_READY, &message); + if (err != nullptr) { + return err; + } + + const char* buffer = nullptr; + size_t byte_size = 0; + err = TRITONSERVER_MessageSerializeToJson(message, &buffer, &byte_size); + if (err != nullptr) { + TRITONSERVER_MessageDelete(message); + return err; + } + const std::string index_json(buffer, byte_size); + TRITONSERVER_MessageDelete(message); + + rapidjson::Document index_doc; + index_doc.Parse(index_json.data(), index_json.size()); + if (index_doc.HasParseError()) { + const std::string parse_err = std::string("failed to parse model index JSON: ") + rapidjson::GetParseError_En(index_doc.GetParseError()) + " at offset " + std::to_string(index_doc.GetErrorOffset()); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, parse_err.c_str()); + } + if (!index_doc.IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "model index JSON root is not an array"); + } + + out->reserve(static_cast(index_doc.Size())); + for (rapidjson::SizeType i = 0; i < index_doc.Size(); ++i) { + const rapidjson::Value& o = index_doc[i]; + if (!o.IsObject() || !o.HasMember("name")) { + continue; + } + const rapidjson::Value& n = o["name"]; + if (!n.IsString()) { + continue; + } + out->emplace(n.GetString()); + } + + return nullptr; +} + +TRITONSERVER_Error* AppendRowToNamedDoubleBuffers(NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, + const std::string& vector_name, const std::vector& row, size_t feature_count) { + if (buffers == nullptr || feature_counts == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer pointers are null"); + } + if (row.size() != feature_count || feature_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "row width does not match feature_count"); + } + + auto fc_it = feature_counts->find(vector_name); + if (fc_it == feature_counts->end()) { + (*feature_counts)[vector_name] = feature_count; + } else if (fc_it->second != feature_count) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "inconsistent feature_count for model buffer"); + } + + std::vector& buf = (*buffers)[vector_name]; + buf.insert(buf.end(), row.begin(), row.end()); + return nullptr; +} + +TRITONSERVER_Error* BuildMultiInferRequestDocument(const NamedDoubleBuffers& buffers, const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc) { + if (out_doc == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "output document pointer is null"); + } + + std::vector names; + names.reserve(buffers.size()); + for (const auto& kv : buffers) { + names.push_back(kv.first); + } + std::sort(names.begin(), names.end()); + + rapidjson::Document doc(rapidjson::kObjectType); + auto& alloc = doc.GetAllocator(); + rapidjson::Value requests(rapidjson::kArrayType); + + for (const std::string& model_name : names) { + auto bc_it = buffers.find(model_name); + auto fc_it = feature_counts.find(model_name); + if (bc_it == buffers.end() || fc_it == feature_counts.end()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INTERNAL, "internal buffer map inconsistency"); + } + const size_t feature_count = fc_it->second; + if (feature_count == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "zero feature_count for model"); + } + const std::vector& flat = bc_it->second; + if (flat.size() % feature_count != 0 || flat.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "buffer length is not a multiple of feature_count"); + } + const size_t rows = flat.size() / feature_count; + + rapidjson::Value req(rapidjson::kObjectType); + req.AddMember("model_name", rapidjson::Value(model_name.c_str(), static_cast(model_name.size()), alloc).Move(), alloc); + + rapidjson::Value data(rapidjson::kArrayType); + data.Reserve(static_cast(flat.size()), alloc); + for (double v : flat) { + data.PushBack(v, alloc); + } + + rapidjson::Value shape(rapidjson::kArrayType); + shape.PushBack(static_cast(rows), alloc); + shape.PushBack(static_cast(feature_count), alloc); + + rapidjson::Value input0(rapidjson::kObjectType); + input0.AddMember("name", "input__0", alloc); + input0.AddMember("datatype", "FP32", alloc); + input0.AddMember("shape", shape, alloc); + input0.AddMember("data", data, alloc); + + rapidjson::Value inputs(rapidjson::kArrayType); + inputs.PushBack(input0, alloc); + + rapidjson::Value outputs(rapidjson::kArrayType); + rapidjson::Value out0(rapidjson::kObjectType); + out0.AddMember("name", "output__0", alloc); + outputs.PushBack(out0, alloc); + + req.AddMember("inputs", inputs, alloc); + req.AddMember("outputs", outputs, alloc); + requests.PushBack(req, alloc); + } + + doc.AddMember("requests", requests, alloc); + *out_doc = std::move(doc); + return nullptr; +} + +TRITONSERVER_Error* GenerateInputVectors(const rapidjson::Document& doc, TRITONSERVER_Server* server, rapidjson::Document* out_doc) { + if (out_doc == nullptr || server == nullptr) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "invalid argument"); + } + if (!doc.IsObject() || !doc.HasMember("imps") || !doc["imps"].IsArray()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "expected object with imps array"); + } + + const CampaignToFeatureMappings* cmap = ActiveCampaignToFeatureMappings(); + if (cmap == nullptr || cmap->empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_UNAVAILABLE, "campaign feature mappings are not loaded"); + } + + std::unordered_set ready_model_names; + TRITONSERVER_Error* err = GetReadyModelNames(server, &ready_model_names); + if (err != nullptr) { + return err; + } + if (ready_model_names.empty()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "no ready models reported by server"); + } + + NamedDoubleBuffers buffers; + ModelNameToFeatureCount counts; + + const rapidjson::Value& imps = doc["imps"]; + for (rapidjson::SizeType ii = 0; ii < imps.Size(); ++ii) { + const rapidjson::Value& imp = imps[ii]; + if (!imp.IsObject()) { + continue; + } + if (!imp.HasMember("camps") || !imp["camps"].IsArray() || imp["camps"].Size() == 0) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each impression must include a non-empty camps array"); + } + const rapidjson::Value& camps = imp["camps"]; + for (rapidjson::SizeType ci = 0; ci < camps.Size(); ++ci) { + const rapidjson::Value& camp = camps[ci]; + if (!camp.IsObject() || !camp.HasMember("cid") || !camp["cid"].IsInt()) { + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, "each camp must be an object with integer 'cid'"); + } + const int32_t campaign_id = camp["cid"].GetInt(); + + auto cmap_it = cmap->find(campaign_id); + if(cmap_it == cmap->end()) { + cmap_it = cmap->find(0); + } + if (cmap_it == cmap->end()) { + const std::string unknown_campaign = std::string("unknown campaign_id ") + std::to_string(campaign_id); + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, unknown_campaign.c_str()); + } + const std::string& model_name = cmap_it->second.model_name; + const std::vector& feature_sequence = cmap_it->second.feature_sequence; + const FeatureMappingTables& tables = cmap_it->second.feature_mapping; + + if (ready_model_names.find(model_name) == ready_model_names.end()) { + const std::string not_ready = "model " + model_name + " not ready"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, not_ready.c_str()); + } + + std::vector row(feature_sequence.size(), 0.0); + int adsize_idx = -1; + + for (size_t fi = 0; fi < feature_sequence.size(); ++fi) { + const std::string& feature = feature_sequence[fi]; + const char* fkey = feature.c_str(); + + if (feature == "adsize") { + adsize_idx = static_cast(fi); + continue; + } + + const rapidjson::Value* src = nullptr; + if (feature == "cookie" || feature == "rnk") { + if (camp.HasMember(fkey)) { + src = &camp[fkey]; + } + } + else if(feature == "campid") { + if (camp.HasMember("cid")) { + src = &camp["cid"]; + } + } + else { + if (imp.HasMember(fkey)) { + src = &imp[fkey]; + } + } + + if (src == nullptr) { + const std::string missing_field = std::string("missing JSON field for feature '") + feature + "'"; + return TRITONSERVER_ErrorNew(TRITONSERVER_ERROR_INVALID_ARG, missing_field.c_str()); + } + int idx = FeatureIdxFromJsonValue(feature.c_str(), *src, &tables); + row[fi] = static_cast(idx); + } + + if (adsize_idx >= 0 && camp.HasMember("adsize") && camp["adsize"].IsArray()) { + const rapidjson::Value& adsize = camp["adsize"]; + for (rapidjson::SizeType ai = 0; ai < adsize.Size(); ++ai) { + const rapidjson::Value& adsize_item = adsize[ai]; + int mapped = FeatureIdxFromJsonValue("adsize", adsize_item, &tables); + row[static_cast(adsize_idx)] = static_cast(mapped); + err = AppendRowToNamedDoubleBuffers(&buffers, &counts, model_name, row, feature_sequence.size()); + if (err != nullptr) { + return err; + } + } + } + } + } + err = BuildMultiInferRequestDocument(buffers, counts, out_doc); + return err; + } +} +} // namespace triton::server +#endif // TRITON_ENABLE_MYSQL_ODBC diff --git a/src/transform.h b/src/transform.h new file mode 100644 index 0000000000..f481098d55 --- /dev/null +++ b/src/transform.h @@ -0,0 +1,70 @@ +// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of NVIDIA CORPORATION nor the names of its +// contributors may be used to endorse or promote products derived +// from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +// OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include + +#include "triton/core/tritonserver.h" + +namespace triton { namespace server { + +// Parses `json` into a document. If built with `TRITON_ENABLE_MYSQL_ODBC` and +// `server` is non-null and the root object contains `"imps"`, replaces +// `*out_doc` with a multi-infer request built from impressions; otherwise +// stores the parsed root in `*out_doc`. +// Returns nullptr on success, or a TRITONSERVER_Error that the caller must +// delete with TRITONSERVER_ErrorDelete on failure. +TRITONSERVER_Error* ParseRequest(const std::string& json, TRITONSERVER_Server* server, rapidjson::Document* out_doc); + +#ifdef TRITON_ENABLE_MYSQL_ODBC + +// Builds `*out_doc` as a `POST /v2/multi_infer` body from `doc` (imps/camps). +TRITONSERVER_Error* GenerateInputVectors( + const rapidjson::Document& doc, TRITONSERVER_Server* server, + rapidjson::Document* out_doc); + +TRITONSERVER_Error* GetReadyModelNames( + TRITONSERVER_Server* server, std::unordered_set* out); + +using NamedDoubleBuffers = std::unordered_map>; +using ModelNameToFeatureCount = std::unordered_map; + +TRITONSERVER_Error* AppendRowToNamedDoubleBuffers( + NamedDoubleBuffers* buffers, ModelNameToFeatureCount* feature_counts, + const std::string& vector_name, const std::vector& row, + size_t feature_count); + +TRITONSERVER_Error* BuildMultiInferRequestDocument( + const NamedDoubleBuffers& buffers, + const ModelNameToFeatureCount& feature_counts, rapidjson::Document* out_doc); + +#endif // TRITON_ENABLE_MYSQL_ODBC + +}} // namespace triton::server