diff --git a/Dockerfile.sdk b/Dockerfile.sdk index b2181abe6e..d6eed1e363 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:26.05-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:26.06-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server diff --git a/README.md b/README.md index fdb6b2a5bf..a913f11478 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ >[!WARNING] >You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.69.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 26.05 container release on NVIDIA GPU Cloud (NGC). +>towards the next release. The current release is version [2.70.0](https://github.com/triton-inference-server/server/releases/latest) +>and corresponds to the 26.06 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server @@ -90,16 +90,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r26.05 https://github.com/triton-inference-server/server.git +git clone -b r26.06 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:26.05-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:26.06-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:26.05-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:26.06-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/build.py b/build.py index ca8f2c9d7a..e945528923 100755 --- a/build.py +++ b/build.py @@ -32,6 +32,7 @@ import os.path import pathlib import platform +import re import stat import subprocess import sys @@ -75,8 +76,8 @@ "triton_container_version": "26.07dev", "upstream_container_version": "26.05", "ort_version": "1.24.4", - "ort_openvino_version": "2026.1.0", - "standalone_openvino_version": "2026.1.0", + "ort_openvino_version": "2026.2.0", + "standalone_openvino_version": "2026.2.0", "dcgm_version": "4.5.3-1", "rhel_py_version": "3.12.3", } @@ -921,6 +922,7 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ + auditwheel \\ build \\ wheel \\ setuptools \\ @@ -1034,6 +1036,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap): && rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade \\ + auditwheel \\ build \\ docker \\ virtualenv \\ @@ -1171,10 +1174,21 @@ def create_dockerfile_linux( WORKDIR /opt/tritonserver COPY NVIDIA_Deep_Learning_Container_License.pdf . -RUN find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonserver-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] && \\ - find /opt/tritonserver/python -maxdepth 1 -type f -name \\ - "tritonfrontend-*.whl" | xargs -I {{}} pip install --upgrade {{}}[{FLAGS.triton_wheels_dependencies_group}] +# TRI-1118 — fail fast if either tritonserver or tritonfrontend wheel is +# missing from /opt/tritonserver/python. The legacy `find | xargs -I` is +# a silent no-op when find returns zero matches: xargs runs the command +# zero times and the layer succeeds, masking the gap until something +# downstream (a wheel publish job, or pip install tritonfrontend) +# discovers nothing was actually installed. Check existence first. +RUN set -e; \\ + for pkg in tritonserver tritonfrontend; do \\ + wheels=$(find /opt/tritonserver/python -maxdepth 1 -type f -name "${{pkg}}-*.whl"); \\ + if [ -z "$wheels" ]; then \\ + echo "ERROR: ${{pkg}}-*.whl missing from /opt/tritonserver/python -- build did not stage the wheel into the image" >&2; \\ + exit 1; \\ + fi; \\ + printf '%s\\n' "$wheels" | xargs -I {{}} pip install --upgrade "{{}}[{FLAGS.triton_wheels_dependencies_group}]"; \\ + done RUN pip3 install -r python/openai/requirements.txt @@ -1301,6 +1315,7 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach libgoogle-perftools-dev \\ libjemalloc-dev \\ libnuma-dev \\ + libssl-dev \\ wget \\ {backend_dependencies} \\ python3-pip \\ @@ -1664,6 +1679,25 @@ def create_docker_build_script(script_name, container_install_dir, container_ci_ ), ] + # TRI-1118 — propagate TRITON_RELEASE_VERSION into the wheel build + # only when it was actually set in main() (release-semantic version + # or explicit --release-version). Dev / pre-release builds leave it + # unset so build_wheel.py reads the in-tree TRITON_VERSION file and + # takes the PEP 817 variant path. + if "TRITON_RELEASE_VERSION" in os.environ: + runargs += [ + "-e", + f"TRITON_RELEASE_VERSION={os.environ['TRITON_RELEASE_VERSION']}", + ] + # TRI-1118 — forward PEP 427 build-tag and PEP 817 nv-part inputs so + # build_wheel.py inside the buildbase container can emit pipeline-correct + # wheel filenames. CUDA_VERSION is deliberately NOT forwarded -- the + # container's own CUDA base image defines it; host CUDA may differ + # (see core/python/build_wheel.py:_detect_cuda_version). + for var in ("CI_PIPELINE_ID", "NVIDIA_UPSTREAM_VERSION", "NVIDIA_BUILD_ID"): + if os.environ.get(var): + runargs += ["-e", f"{var}={os.environ[var]}"] + runargs += ["tritonserver_buildbase"] runargs += ["./cmake_build"] @@ -1787,6 +1821,18 @@ def core_build( # [FIXME] Placing the tritonserver and tritonfrontend wheel files in 'python' for now, # should be uploaded to pip registry to be able to install directly cmake_script.mkdir(os.path.join(install_dir, "python")) + # TRI-1118 — tritonfrontend wheel is built by the triton-server + # sub-build's `frontend-server-wheel` target, but its CMake + # `install(DIRECTORY ${WHEEL_OUT_DIR})` rule doesn't traverse the + # outer install pass, so the wheel never lands in + # repo_install_dir/python/ alongside the core's tritonserver wheel. + # Pull it in from the build tree directly so the subsequent + # `triton*.whl` glob picks it up. Safe no-op when the wheel is + # absent (e.g. downstream builds that disable the frontend). + cmake_script.cmd( + f"find {repo_build_dir} -path '*/wheel/dist/tritonfrontend-*.whl' " + f"-exec cp {{}} {os.path.join(repo_install_dir, 'python')}/ \\;" + ) cmake_script.cp( os.path.join(repo_install_dir, "python", "triton*.whl"), os.path.join(install_dir, "python"), @@ -2466,8 +2512,12 @@ def enable_all(): parser.add_argument( "--release-version", required=False, - default=DEFAULT_TRITON_VERSION_MAP["release_version"], - help="This flag sets the release version for Triton Inference Server to be built. Default: the latest released version.", + default=None, + help="Override the wheel base version (TRI-1118). When set, exported " + "as TRITON_RELEASE_VERSION so build_wheel.py uses it as the bare " + "PEP 440 version. When unset, build.py falls back to --version when " + "it matches X.Y.Z release-semantic, otherwise leaves the env var " + "unset and lets the in-tree TRITON_VERSION file rule (PEP 817 path).", ) parser.add_argument( "--triton-container-version", @@ -2596,6 +2646,22 @@ def enable_all(): if FLAGS.version is None: FLAGS.version = DEFAULT_TRITON_VERSION_MAP["release_version"] + # TRI-1118 — choose whether to export TRITON_RELEASE_VERSION based on + # the resolved Triton version: + # - --release-version explicitly passed -> use it (override) + # - FLAGS.version matches X.Y.Z -> propagate FLAGS.version + # - anything else (dev / pre-release / etc) -> leave unset so the + # wheel build reads the + # in-tree TRITON_VERSION + # file (PEP 817 variant). + # The container build forwards this env var via `docker run -e` + # (create_docker_build_script). For --no-container-build, the + # cmake_build subprocess inherits the host env directly. + if FLAGS.release_version is not None: + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.release_version) + elif re.match(r"^\d+\.\d+\.\d+$", FLAGS.version): + os.environ.setdefault("TRITON_RELEASE_VERSION", FLAGS.version) + if FLAGS.build_parallel is None: FLAGS.build_parallel = multiprocessing.cpu_count() * 2 diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index c94f832aa8..81700622b4 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index bd360e7955..8a9cc7a56d 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.69.0 +appVersion: 2.70.0 description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index b911db4afd..33e525bceb 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r26.05/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r26.06/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r26.05/README.md + # see https://github.com/triton-inference-server/server/blob/r26.06/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index 9784c9d252..6418ab82c0 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 0e1347f4fd..3c1b93dfe6 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:26.05-py3-sdk + - image: nvcr.io/nvidia/tritonserver:26.06-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index 4b4468d89d..6e166361ae 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -28,8 +28,8 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver export MAJOR_VERSION=2.67 -export MINOR_VERSION=2.69.0 -export NGC_VERSION=26.05-py3 +export MINOR_VERSION=2.70.0 +export NGC_VERSION=26.06-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index d150f0e8d7..d1323108e3 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -28,4 +28,4 @@ apiVersion: v1 appVersion: "2.68" description: Triton Inference Server name: triton-inference-server -version: 2.69.0 +version: 2.70.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 362107e71a..17232cdd2b 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/26.05 -publishedVersion: '2.69.0' +modelRepositoryPath: gs://triton_sample_models/26.06 +publishedVersion: '2.70.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 26.05-py3 + tag: 26.06-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 4c312c9880..6a68da95d6 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.70.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index ccf3b157c4..2e95769c0c 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.69.0' + publishedVersion: '2.70.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/26.05 + default: gs://triton_sample_models/26.06 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index fff7466da4..6c6f26acbd 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:26.05-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:26.06-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/26.05/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/26.06/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/26.05/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/26.06/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 3d788f3f17..8b83d1cf98 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index df5d60066d..3ef505aa92 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:26.05-py3 + imageName: nvcr.io/nvidia/tritonserver:26.06-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/customization_guide/compose.md b/docs/customization_guide/compose.md index e922d27fbe..45eb95eda0 100644 --- a/docs/customization_guide/compose.md +++ b/docs/customization_guide/compose.md @@ -46,8 +46,8 @@ The `compose.py` script can be found in the Simply clone the repository and run `compose.py` to create a custom container. Note: Created container version will depend on the branch that was cloned. For example branch - [r26.05](https://github.com/triton-inference-server/server/tree/r26.05) -should be used to create a image based on the NGC 26.05 Triton release. + [r26.06](https://github.com/triton-inference-server/server/tree/r26.06) +should be used to create a image based on the NGC 26.06 Triton release. `compose.py` provides `--backend`, `--repoagent` options that allow you to specify which backends and repository agents to include in the custom image. @@ -78,20 +78,20 @@ For example, running ``` python3 compose.py --backend pytorch --repoagent checksum ``` -on branch [r26.05](https://github.com/triton-inference-server/server/tree/r26.05) pulls: -- `min` container `nvcr.io/nvidia/tritonserver:26.05-py3-min` -- `full` container `nvcr.io/nvidia/tritonserver:26.05-py3` +on branch [r26.06](https://github.com/triton-inference-server/server/tree/r26.06) pulls: +- `min` container `nvcr.io/nvidia/tritonserver:26.06-py3-min` +- `full` container `nvcr.io/nvidia/tritonserver:26.06-py3` Alternatively, users can specify the version of Triton container to pull from any branch by either: 1. Adding flag `--container-version ` to branch ``` -python3 compose.py --backend pytorch --repoagent checksum --container-version 26.05 +python3 compose.py --backend pytorch --repoagent checksum --container-version 26.06 ``` 2. Specifying `--image min, --image full,`. The user is responsible for specifying compatible `min` and `full` containers. ``` -python3 compose.py --backend pytorch --repoagent checksum --image min,nvcr.io/nvidia/tritonserver:26.05-py3-min --image full,nvcr.io/nvidia/tritonserver:26.05-py3 +python3 compose.py --backend pytorch --repoagent checksum --image min,nvcr.io/nvidia/tritonserver:26.06-py3-min --image full,nvcr.io/nvidia/tritonserver:26.06-py3 ``` Method 1 and 2 will result in the same composed container. Furthermore, `--image` flag overrides the `--container-version` flag when both are specified. @@ -102,8 +102,8 @@ Note: 2. vLLM and TensorRT-LLM backends are currently not supported backends for `compose.py`. If you want to build additional backends on top of these backends, it would be better to [build it yourself](#build-it-yourself) by using -`nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3` or -`nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3` as a `min` container. +`nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3` or +`nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3` as a `min` container. ### CPU-only container composition diff --git a/docs/getting_started/llm.md b/docs/getting_started/llm.md index 6ea9a5aa33..2ae5f82290 100644 --- a/docs/getting_started/llm.md +++ b/docs/getting_started/llm.md @@ -282,7 +282,7 @@ The above needs to be done manually with your favorite editor. Once finished, pl -v $(pwd)/all_models:/opt/all_models \ -v $(pwd)/scripts:/opt/scripts \ -v $(pwd)/Phi-3-mini-4k-instruct:/opt/Phi-3-mini-4k-instruct \ - nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3 # Launch Server python3 ../scripts/launch_triton_server.py --model_repo ../all_models/inflight_batcher_llm --world_size 1 @@ -308,7 +308,7 @@ The above needs to be done manually with your favorite editor. Once finished, pl - export RELEASE="26.05" + export RELEASE="26.06" docker run -it --net=host --gpus '"device=0"' nvcr.io/nvidia/tritonserver:${RELEASE}-py3-sdk 17. ## Download the Phi-3 tokenizer diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index 5fdb71adee..16ac0c1e00 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,6 +38,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 26.06 | nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3 | Python 3.12.3 | 2.10.0a0+b4e4ee81d3.nv25.12 | 10.14.1.48 | 1.2.1 | 13.1.0.036 | 590.44.01 | 14.24 GB | | 26.05 | nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 | Python 3.12.3 | 2.10.0a0+b4e4ee81d3.nv25.12 | 10.14.1.48 | 1.2.1 | 13.1.0.036 | 590.44.01 | 14.22 GB | | 26.04 | nvcr.io/nvidia/tritonserver:26.04-trtllm-python-py3 | Python 3.12.3 | 2.10.0a0+b4e4ee81d3.nv25.12 | 10.14.1.48 | 1.2.1 | 13.1.0.036 | 590.44.01 | 14.22 GB | | 26.03 | nvcr.io/nvidia/tritonserver:26.03-trtllm-python-py3 | Python 3.12.3 | 2.10.0a0+b4e4ee81d3.nv25.12 | 10.14.1.48 | 1.2.0 | 13.1.0.036 | 590.44.01 | 14.18 GB | @@ -69,6 +70,7 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | +| 26.06 | nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3 | Python 3.12.3 | 0.22.1+7b9cb5b7.nv26.6.55098374 | 13.3.0.035 | 610.43.02 | 10.21 GB | | 26.05 | nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 | Python 3.12.3 | 0.19.0+6bc3197f.nv26.04.48761268 | 13.2.1.009 | 595.58.03 | 9.3G | | 26.04 | nvcr.io/nvidia/tritonserver:26.04-vllm-python-py3 | Python 3.12.3 | 0.19.0+6bc3197f.nv26.04.48761268 | 13.2.1.009 | 595.58.03 | 9.09G | | 26.03 | nvcr.io/nvidia/tritonserver:26.03-vllm-python-py3 | Python 3.12.3 | 0.17.1+fb2e3ab6.nv26.3.46332470.cu132 | 13.2.0.046 | 595.45.04 | 9.22G | @@ -100,6 +102,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 26.06 | 1.24.4 | | 26.05 | 1.24.4 | | 26.04 | 1.24.4 | | 26.03 | 1.24.2 | diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 19fc0f22f3..6c15ceb35a 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 26.05](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-26-05.html#rel-26-05) +# [Triton Inference Server Release 26.06](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-26-06.html#rel-26-06) -The Triton Inference Server container image, release 26.05, is available +The Triton Inference Server container image, release 26.06, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/docs/user_guide/performance_tuning.md b/docs/user_guide/performance_tuning.md index ff837a4629..4d5eee16a8 100644 --- a/docs/user_guide/performance_tuning.md +++ b/docs/user_guide/performance_tuning.md @@ -235,7 +235,7 @@ with a `tritonserver` binary. ```bash # Start server container -docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-server nvcr.io/nvidia/tritonserver:26.05-py3 +docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-server nvcr.io/nvidia/tritonserver:26.06-py3 # Start serving your models tritonserver --model-repository=/mnt/models @@ -284,7 +284,7 @@ by setting the `-u` flag, such as `perf_analyzer -m densenet_onnx -u ```bash # Start the SDK container interactively -docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-client nvcr.io/nvidia/tritonserver:26.05-py3-sdk +docker run -ti --rm --gpus=all --network=host -v $PWD:/mnt --name triton-client nvcr.io/nvidia/tritonserver:26.06-py3-sdk # Benchmark model being served from step 3 perf_analyzer -m densenet_onnx --concurrency-range 1:4 diff --git a/python/openai/README.md b/python/openai/README.md index dbfd4b7792..8dd7b55844 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -46,7 +46,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -355,7 +355,7 @@ Currently, OpenAI-Compatible Frontend supports loading embedding models and embe docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:26.05-vllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: @@ -451,7 +451,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ -e TRTLLM_ORCHESTRATOR=1 \ - nvcr.io/nvidia/tritonserver:26.05-trtllm-python-py3 + nvcr.io/nvidia/tritonserver:26.06-trtllm-python-py3 ``` 2. Install dependencies inside the container: diff --git a/qa/common/gen_common.py b/qa/common/gen_common.py index db0869ef38..00d23e1e54 100644 --- a/qa/common/gen_common.py +++ b/qa/common/gen_common.py @@ -146,6 +146,28 @@ def np_to_torch_dtype(np_dtype): return None +def trt_set_dynamic_range(tensor, lo, hi): + """Set ITensor.dynamic_range on TRT versions that support it. + + Removed in TensorRT 11+ (strongly-typed networks). Silently skip on + versions where the attribute is gone so the QA model-gen scripts stay + compatible with both old and new TRT.""" + try: + tensor.dynamic_range = (lo, hi) + except AttributeError: + pass # ITensor.dynamic_range removed in TensorRT 11+ (strongly-typed) + + +def trt_cast_tensor(network, tensor, target_dtype): + """Insert an explicit dtype cast that works on both TRT 8.5+ (add_cast) + and older TRT (add_identity + set_output_type). Returns the cast layer.""" + if hasattr(network, "add_cast"): + return network.add_cast(tensor, target_dtype) + layer = network.add_identity(tensor) + layer.set_output_type(0, target_dtype) + return layer + + def openvino_save_model(model_version_dir, model): import openvino as ov diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 4d491fa2a1..658f272f86 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=26.05} +TRITON_VERSION=${TRITON_VERSION:=26.06} # Set the CUDA device to use NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_dyna_sequence_implicit_models.py b/qa/common/gen_qa_dyna_sequence_implicit_models.py index c69ca28eab..784e63ffb5 100755 --- a/qa/common/gen_qa_dyna_sequence_implicit_models.py +++ b/qa/common/gen_qa_dyna_sequence_implicit_models.py @@ -30,7 +30,12 @@ import os import numpy as np -from gen_common import np_to_model_dtype, np_to_onnx_dtype, np_to_trt_dtype +from gen_common import ( + np_to_model_dtype, + np_to_onnx_dtype, + np_to_trt_dtype, + trt_set_dynamic_range, +) FLAGS = None np_dtype_string = np.dtype(object) @@ -388,7 +393,10 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -527,7 +535,10 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype, so this was a no-op. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) input_state_cond_temp = network.add_elementwise( ready0, not_start.get_output(0), trt.ElementWiseOperation.SUM @@ -552,11 +563,19 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0_state.get_output(0).dtype = trt_dtype + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) in_state0.allowed_formats = 1 << int(trt_memory_format) @@ -566,21 +585,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - out0_state.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(out0_state, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py index 1a26890f32..c4cb5bf26a 100755 --- a/qa/common/gen_qa_dyna_sequence_models.py +++ b/qa/common/gen_qa_dyna_sequence_models.py @@ -36,6 +36,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -101,15 +102,26 @@ def create_plan_shape_tensor_modelfile( resized_out0 = resize_layer.get_output(0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # ITensor.dtype setter removed in TRT 11; shape/elementwise/resize + # outputs already have the correct dtype. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt.int32 + try: + out0.dtype = trt.int32 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) shape_in0.allowed_formats = 1 << int(trt_memory_format) @@ -121,20 +133,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - dummy_in0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(dummy_in0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -353,7 +367,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -361,21 +380,23 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - end0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - corrid0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(end0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) + trt_set_dynamic_range(corrid0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py index 426d939d9e..bc87333205 100755 --- a/qa/common/gen_qa_identity_models.py +++ b/qa/common/gen_qa_identity_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -552,14 +553,18 @@ def create_plan_dynamic_rf_modelfile( out_node = network.add_identity(in_node) out_node.get_output(0).name = "OUTPUT{}".format(io_num) - out_node.get_output(0).dtype = trt_dtype + # Identity preserves input dtype; the ITensor.dtype setter was + # removed in TensorRT 11. Older TRT versions still accept it. + try: + out_node.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -583,14 +588,17 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT{}".format(io_num), min_shape, opt_shape, max_shape) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -668,18 +676,26 @@ def create_plan_shape_tensor_modelfile( dummy_out_node.name = "DUMMY_OUTPUT{}".format(io_num) - dummy_out_node.dtype = trt_dtype + # The ITensor.dtype setter was removed in TensorRT 11; resize and + # shape layers already produce the correct dtype, so suppress the + # AttributeError instead of changing the older-TRT behavior. + try: + dummy_out_node.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(dummy_out_node) dummy_out_node.allowed_formats = 1 << int(trt_memory_format) - out_node.get_output(0).dtype = trt.int64 + try: + out_node.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(out_node.get_output(0)) out_node.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in_node.dynamic_range = (-128.0, 127.0) - out_node.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in_node, -128.0, 127.0) + trt_set_dynamic_range(out_node.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_prefix = [] @@ -707,14 +723,17 @@ def create_plan_shape_tensor_modelfile( config.add_optimization_profile(profile) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config.flags = flags diff --git a/qa/common/gen_qa_implicit_models.py b/qa/common/gen_qa_implicit_models.py index e042e40342..d3b28da96a 100755 --- a/qa/common/gen_qa_implicit_models.py +++ b/qa/common/gen_qa_implicit_models.py @@ -37,6 +37,7 @@ np_to_onnx_dtype, np_to_torch_dtype, np_to_trt_dtype, + trt_set_dynamic_range, ) FLAGS = None @@ -923,7 +924,11 @@ def create_plan_modelfile(models_dir, model_version, max_batch, dtype, shape): not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1033,7 +1038,11 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) not_start = network.add_elementwise( constant_1.get_output(0), start0, trt.ElementWiseOperation.SUB ) - not_start.set_output_type(0, trt_dtype) + # set_output_type was removed from all layers in TensorRT 11; the + # elementwise output already has trt_dtype (both inputs do), so this + # call was a no-op on modern TRT. Guard for older versions. + if hasattr(not_start, "set_output_type"): + not_start.set_output_type(0, trt_dtype) internal_state = network.add_elementwise( in_state0, not_start.get_output(0), trt.ElementWiseOperation.PROD ) @@ -1049,8 +1058,16 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0_state.get_output(0).name = "OUTPUT_STATE" network.mark_output(out0_state.get_output(0)) - out0.get_output(0).dtype = trt_dtype - out0_state.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output dtype + # already matches trt_dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ + try: + out0_state.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -1058,20 +1075,22 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in_state0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in_state0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index a9763f3c83..7fae1a0b23 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -66,7 +66,7 @@ log_message.status "Changing working directory to the script directory to: " "${ cd ${TRITON_MDLS_BASE_SCRIPT_DIR} log_message.status "define: default values" -TRITON_VERSION=${TRITON_VERSION:=26.05} +TRITON_VERSION=${TRITON_VERSION:=26.06} ONNX_VERSION=1.20.1 ONNX_OPSET=0 OPENVINO_VERSION=2024.5.0 @@ -628,7 +628,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: openvino.ubuntu.$CI_JOB_ID" enroot create --name openvino.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: openvino.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR openvino.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$OPENVINOSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${OPENVINOSCRIPT} failed" exit 1 @@ -638,7 +638,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: onnxruntime.ubuntu.$CI_JOB_ID" enroot create --name onnxruntime.ubuntu.$CI_JOB_ID /tmp/ubuntu.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: onnxruntime.ubuntu.$CI_JOB_ID" - enroot start --root --rw -m /tmp:/tmp onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT + enroot start --root --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR onnxruntime.ubuntu.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$ONNXSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${ONNXSCRIPT} failed" exit 1 @@ -655,7 +655,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: pytorch.$CI_JOB_ID" enroot create --name pytorch.$CI_JOB_ID /tmp/pytorch.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: pytorch.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR pytorch.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TORCHSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TORCHSCRIPT} failed" exit 1 @@ -666,7 +666,7 @@ elif [ "$TRITON_MODELS_USE_ENROOT" -eq 1 ] && which enroot ; then log_message.status "enroot create: tensorrt.$CI_JOB_ID" enroot create --name tensorrt.$CI_JOB_ID /tmp/tensorrt.$CI_JOB_ID.enroot.sqsh log_message.info "enroot start: tensorrt.$CI_JOB_ID" - enroot start --rw -m /tmp:/tmp tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT + enroot start --rw -m /tmp:/tmp -e TRITON_GENSRCDIR=$TRITON_MDLS_SRC_DIR tensorrt.$CI_JOB_ID bash -xe $TRITON_MDLS_SRC_DIR/$TRTSCRIPT if [ $? -ne 0 ]; then log_message.error "enroot start: ${TRTSCRIPT} failed" exit 1 diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py index b50e0015ed..68217a3b5d 100755 --- a/qa/common/gen_qa_models.py +++ b/qa/common/gen_qa_models.py @@ -40,6 +40,8 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_cast_tensor, + trt_set_dynamic_range, ) FLAGS = None @@ -95,11 +97,11 @@ def create_plan_dynamic_rf_modelfile( # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) if tu.support_trt_uint8(): if trt_input_dtype == trt.uint8: - in0_cast = network.add_identity(in0) - in0_cast.set_output_type(0, trt.float32) + # TensorRT 11 removed set_output_type on identity layers; the + # equivalent on TRT 8.5+ is add_cast. + in0_cast = trt_cast_tensor(network, in0, trt.float32) in0 = in0_cast.get_output(0) - in1_cast = network.add_identity(in1) - in1_cast.set_output_type(0, trt.float32) + in1_cast = trt_cast_tensor(network, in1, trt.float32) in1 = in1_cast.get_output(0) add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) @@ -107,37 +109,32 @@ def create_plan_dynamic_rf_modelfile( out0 = add if not swap else sub out1 = sub if not swap else add - # uint8 conversion after operations - # FIXME: Remove support check when jetson supports TRT 8.5 (DLIS-4256) - if tu.support_trt_uint8(): - if trt_output0_dtype == trt.uint8: - out0 = network.add_identity(out0.get_output(0)) - out0.set_output_type(0, trt.uint8) - if trt_output1_dtype == trt.uint8: - out1 = network.add_identity(out1.get_output(0)) - out1.set_output_type(0, trt.uint8) + # TRT 11 strongly-typed networks: ITensor.dtype setter no longer coerces + # output dtype, so insert an explicit cast whenever the elementwise op's + # natural output dtype differs from the declared output dtype. Covers + # both float<->float and uint8 cases. + if out0.get_output(0).dtype != trt_output0_dtype: + out0 = trt_cast_tensor(network, out0.get_output(0), trt_output0_dtype) + if out1.get_output(0).dtype != trt_output1_dtype: + out1 = trt_cast_tensor(network, out1.get_output(0), trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype - in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -159,15 +156,19 @@ def create_plan_dynamic_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags @@ -411,27 +412,31 @@ def create_plan_fixed_rf_modelfile( out0 = add if not swap else sub out1 = sub if not swap else add + # TRT 11 strongly-typed networks: ITensor.dtype setter no longer coerces + # output dtype, so insert an explicit cast whenever the elementwise op's + # natural output dtype differs from the declared output dtype. + if out0.get_output(0).dtype != trt_output0_dtype: + out0 = trt_cast_tensor(network, out0.get_output(0), trt_output0_dtype) + if out1.get_output(0).dtype != trt_output1_dtype: + out1 = trt_cast_tensor(network, out1.get_output(0), trt_output1_dtype) + out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.get_output(0).dtype = trt_output0_dtype - out1.get_output(0).dtype = trt_output1_dtype - in0.allowed_formats = 1 << int(trt_memory_format) in1.allowed_formats = 1 << int(trt_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) config = builder.create_builder_config() min_shape = [] @@ -450,15 +455,19 @@ def create_plan_fixed_rf_modelfile( profile.set_shape("INPUT0", min_shape, opt_shape, max_shape) profile.set_shape("INPUT1", min_shape, opt_shape, max_shape) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py index f8d89a5f9e..6de55c3ea5 100755 --- a/qa/common/gen_qa_sequence_models.py +++ b/qa/common/gen_qa_sequence_models.py @@ -37,6 +37,7 @@ np_to_torch_dtype, np_to_trt_dtype, openvino_save_model, + trt_set_dynamic_range, ) FLAGS = None @@ -90,15 +91,28 @@ def create_plan_shape_tensor_modelfile( shape_out0 = network.add_shape(resized_out0) shape_out0.get_output(0).name = "SHAPE_OUTPUT" - shape_out0.get_output(0).dtype = trt.int64 + # The ITensor.dtype setter was removed in TensorRT 11. The shape, resize + # and elementwise outputs already have the correct dtype, so on TRT 11+ + # this assignment is unnecessary; keep the explicit form for older TRT + # under a guard. + try: + shape_out0.get_output(0).dtype = trt.int64 + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output_for_shapes(shape_out0.get_output(0)) out0.name = "OUTPUT" - out0.dtype = trt_dtype + try: + out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(out0) resized_out0.name = "RESIZED_OUTPUT" - resized_out0.dtype = trt_dtype + try: + resized_out0.dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ network.mark_output(resized_out0) in0.allowed_formats = 1 << int(trt_memory_format) @@ -110,20 +124,22 @@ def create_plan_shape_tensor_modelfile( resized_out0.allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - resized_out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(resized_out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_prefix = [] @@ -306,7 +322,12 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).name = "OUTPUT" network.mark_output(out0.get_output(0)) - out0.get_output(0).dtype = trt_dtype + # ITensor.dtype setter removed in TRT 11; elementwise output already has + # the correct dtype. + try: + out0.get_output(0).dtype = trt_dtype + except AttributeError: + pass # ITensor.dtype setter removed in TensorRT 11+ in0.allowed_formats = 1 << int(trt_memory_format) start0.allowed_formats = 1 << int(trt_memory_format) @@ -314,19 +335,21 @@ def create_plan_rf_modelfile(models_dir, model_version, max_batch, dtype, shape) out0.get_output(0).allowed_formats = 1 << int(trt_memory_format) if trt_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - out0.dynamic_range = (-128.0, 127.0) - start0.dynamic_range = (-128.0, 127.0) - ready0.dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(out0, -128.0, 127.0) + trt_set_dynamic_range(start0, -128.0, 127.0) + trt_set_dynamic_range(ready0, -128.0, 127.0) flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) - if trt_dtype == trt.int8: + if trt_dtype == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif trt_dtype == trt.float16: + elif trt_dtype == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) min_shape = [] diff --git a/qa/common/gen_qa_trt_format_models.py b/qa/common/gen_qa_trt_format_models.py index 5f2cadd69e..fc59d7c9e1 100755 --- a/qa/common/gen_qa_trt_format_models.py +++ b/qa/common/gen_qa_trt_format_models.py @@ -32,7 +32,7 @@ import numpy as np import tensorrt as trt import test_util as tu -from gen_common import np_to_model_dtype, np_to_trt_dtype +from gen_common import np_to_model_dtype, np_to_trt_dtype, trt_set_dynamic_range np_dtype_string = np.dtype(object) @@ -98,30 +98,35 @@ def create_plan_modelfile( add = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUM) sub = network.add_elementwise(in0, in1, trt.ElementWiseOperation.SUB) - out0 = network.add_identity(add.get_output(0)) - out1 = network.add_identity(sub.get_output(0)) + # TRT 11 removed Layer.set_output_type; on modern TRT use add_cast to + # produce the desired output dtype. On older TRT, fall back to the + # original identity + set_output_type pattern. + if hasattr(network, "add_cast"): + out0 = network.add_cast(add.get_output(0), trt_output0_dtype) + out1 = network.add_cast(sub.get_output(0), trt_output1_dtype) + else: + out0 = network.add_identity(add.get_output(0)) + out1 = network.add_identity(sub.get_output(0)) + out0.set_output_type(0, trt_output0_dtype) + out1.set_output_type(0, trt_output1_dtype) out0.get_output(0).name = "OUTPUT0" out1.get_output(0).name = "OUTPUT1" network.mark_output(out0.get_output(0)) network.mark_output(out1.get_output(0)) - out0.set_output_type(0, trt_output0_dtype) - out1.set_output_type(0, trt_output1_dtype) - in0.allowed_formats = 1 << int(trt_input_memory_format) in1.allowed_formats = 1 << int(trt_input_memory_format) out0.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) out1.get_output(0).allowed_formats = 1 << int(trt_output_memory_format) if trt_input_dtype == trt.int8: - in0.dynamic_range = (-128.0, 127.0) - in1.dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(in0, -128.0, 127.0) + trt_set_dynamic_range(in1, -128.0, 127.0) if trt_output0_dtype == trt.int8: - out0.get_output(0).dynamic_range = (-128.0, 127.0) + trt_set_dynamic_range(out0.get_output(0), -128.0, 127.0) if trt_output1_dtype == trt.int8: - out1.get_output(0).dynamic_range = (-128.0, 127.0) - + trt_set_dynamic_range(out1.get_output(0), -128.0, 127.0) min_shape = [] opt_shape = [] max_shape = [] @@ -146,14 +151,18 @@ def create_plan_modelfile( # Commenting this because from I/O Formats from TensorRT Developer Guide: # The build will fail if TensorRT cannot build an engine without introducing such reformatting. The failure may happen only for some target platforms, because of what formats are supported by kernels for those platforms. # flags = 1 << int(trt.BuilderFlag.DIRECT_IO) - flags = 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) + # TensorRT 11 removed PREFER_PRECISION_CONSTRAINTS / INT8 / FP16 + # BuilderFlags (strongly-typed networks). Older TRT still has them. + flags = 0 + if hasattr(trt.BuilderFlag, "PREFER_PRECISION_CONSTRAINTS"): + flags |= 1 << int(trt.BuilderFlag.PREFER_PRECISION_CONSTRAINTS) if hasattr(trt.BuilderFlag, "REJECT_EMPTY_ALGORITHMS"): flags |= 1 << int(trt.BuilderFlag.REJECT_EMPTY_ALGORITHMS) datatype_set = set([trt_input_dtype, trt_output0_dtype, trt_output1_dtype]) for dt in datatype_set: - if dt == trt.int8: + if dt == trt.int8 and hasattr(trt.BuilderFlag, "INT8"): flags |= 1 << int(trt.BuilderFlag.INT8) - elif dt == trt.float16: + elif dt == trt.float16 and hasattr(trt.BuilderFlag, "FP16"): flags |= 1 << int(trt.BuilderFlag.FP16) config = builder.create_builder_config() config.flags = flags diff --git a/qa/common/gen_qa_trt_plugin_models.py b/qa/common/gen_qa_trt_plugin_models.py index 9fd23d92a8..9c01de6d6f 100755 --- a/qa/common/gen_qa_trt_plugin_models.py +++ b/qa/common/gen_qa_trt_plugin_models.py @@ -40,6 +40,13 @@ trt.init_libnvinfer_plugins(TRT_LOGGER, "") +# TRT 11 removed the IPluginV2 registry surface (plugin_creator_list). +# Decide V2 vs V3 once at import time and use the same flag for both +# plugin creation and network.add_plugin_v* dispatch. add_plugin_v2 is +# still bound on INetworkDefinition in TRT 11, so hasattr() on the +# network is not a safe gate -- only this registry probe is. +TRT_USES_V3_PLUGINS = not hasattr(trt.get_plugin_registry(), "plugin_creator_list") + def get_trt_plugin(plugin_name): plugin = None @@ -48,8 +55,9 @@ def get_trt_plugin(plugin_name): # branches and V3 on rel-11.0 (and TRT 11 removed the V2 plugin # registry surface). Pick the matching API at runtime. registry = trt.get_plugin_registry() - use_v3 = not hasattr(registry, "plugin_creator_list") - plugin_creators = registry.all_creators if use_v3 else registry.plugin_creator_list + plugin_creators = ( + registry.all_creators if TRT_USES_V3_PLUGINS else registry.plugin_creator_list + ) for plugin_creator in plugin_creators: if (plugin_creator.name == "CustomHardmax") and ( plugin_name == "CustomHardmax" @@ -62,7 +70,7 @@ def get_trt_plugin(plugin_name): if field_collection is None: raise RuntimeError("Plugin not found: " + plugin_name) - if use_v3: + if TRT_USES_V3_PLUGINS: plugin = plugin_creator.create_plugin( name=plugin_name, field_collection=field_collection, @@ -116,16 +124,17 @@ def create_plan_modelfile( input_layer = network.add_input( name="INPUT0", dtype=trt_input_dtype, shape=input_with_batchsize ) - # add_plugin_v2 was removed in TRT 11; add_plugin_v3 has existed since - # TRT 10.0. Pick the API that exists on this TRT install; the plugin - # object returned by get_trt_plugin() is matched to the same version. + # add_plugin_v2 is still bound on INetworkDefinition in TRT 11 but + # rejects IPluginV3 objects; dispatch on the same TRT_USES_V3_PLUGINS + # flag that get_trt_plugin() used to pick the plugin object kind so + # both halves agree on V2 vs V3. plugin_obj = get_trt_plugin(plugin_name) - if hasattr(network, "add_plugin_v2"): - plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) - else: + if TRT_USES_V3_PLUGINS: plugin_layer = network.add_plugin_v3( inputs=[input_layer], shape_inputs=[], plugin=plugin_obj ) + else: + plugin_layer = network.add_plugin_v2(inputs=[input_layer], plugin=plugin_obj) plugin_layer.get_output(0).name = "OUTPUT0" network.mark_output(plugin_layer.get_output(0)) diff --git a/qa/common/test_util.py b/qa/common/test_util.py index 46a42668bf..6f918ded6c 100755 --- a/qa/common/test_util.py +++ b/qa/common/test_util.py @@ -137,6 +137,12 @@ def validate_for_trt_model( # FIXME: Remove this check when jetson supports TRT 8.5 (DLIS-4256) if not support_trt_uint8(): supported_datatypes.remove(np.uint8) + # TRT 11+ removed the implicit-precision INT8 path (BuilderFlag.INT8 + # + dynamic_range); strongly-typed networks require explicit QDQ which + # the QA generators don't emit. Exclude int8 plan models on TRT 11+. + if not support_trt_int8_implicit_precision(): + if np.int8 in supported_datatypes: + supported_datatypes.remove(np.int8) if not input_dtype in supported_datatypes: return False if not output0_dtype in supported_datatypes: @@ -355,6 +361,17 @@ def support_trt_uint8(): return hasattr(trt, "uint8") +def support_trt_int8_implicit_precision(): + """Return True if the installed TensorRT supports the implicit-precision + INT8 path (BuilderFlag.INT8 + per-tensor dynamic_range). Removed in + TensorRT 11+ where strongly-typed networks are mandatory.""" + try: + import tensorrt as trt + except ImportError: + return False + return hasattr(trt.BuilderFlag, "INT8") + + def check_gpus_compute_capability(min_capability): """ Check if all GPUs have a compute capability greater than or equal to the given value. diff --git a/src/python/build_wheel.py b/src/python/build_wheel.py index 27edac6ec5..9ddc21dc5f 100755 --- a/src/python/build_wheel.py +++ b/src/python/build_wheel.py @@ -32,8 +32,23 @@ import shutil import subprocess import sys +import zipfile from tempfile import mkstemp +# ANSI colors for CI log readability (rendered by GitLab CI, harmlessly +# inert in non-ANSI viewers). Suppressed when stderr isn't a TTY and we +# don't appear to be in CI, or when NO_COLOR is set. +if os.environ.get("NO_COLOR") or not sys.stderr.isatty() and not os.environ.get("CI"): + _GREEN = _YELLOW = _CYAN = _RED = _RESET = "" +else: + _GREEN, _YELLOW, _CYAN, _RED, _RESET = ( + "\033[32m", + "\033[33m", + "\033[36m", + "\033[31m", + "\033[0m", + ) + def fail_if(p, msg): if p: @@ -92,64 +107,133 @@ def _detect_cuda_version() -> str | None: return None -def _compose_version(base_version): - """Compose the full wheel version string. - - Appends a PEP 440 local-version segment describing the NVIDIA - container release and CUDA toolkit so consumers can tell an - nv26.04 wheel from an nv26.05 wheel and a cu132 wheel from a - cu128 wheel. All sources are optional; local non-CI builds return - the version unchanged. - """ +def _compose_variant_label(): + """PEP 817 variant label 'nv.cu'. Returns None + if neither input is detectable or the label violates ^[a-z0-9._]{1,16}$.""" nv = ( os.environ.get("NVIDIA_UPSTREAM_VERSION") or os.environ.get("NVIDIA_TRITON_SERVER_VERSION") or os.environ.get("TRITON_CONTAINER_VERSION") ) cuda = _detect_cuda_version() - print( - f"=== Wheel local-version inputs: " - f"NVIDIA_UPSTREAM_VERSION={os.environ.get('NVIDIA_UPSTREAM_VERSION')!r} " - f"NVIDIA_TRITON_SERVER_VERSION={os.environ.get('NVIDIA_TRITON_SERVER_VERSION')!r} " - f"TRITON_CONTAINER_VERSION={os.environ.get('TRITON_CONTAINER_VERSION')!r} " - f"-> nv={nv!r}, cuda={cuda!r}", - file=sys.stderr, - ) - local = [] + parts = [] if nv: - local.append(f"nv{nv}") + parts.append(f"nv{nv}") if cuda: - parts = cuda.split(".") - if len(parts) >= 2 and parts[0].isdigit() and parts[1].isdigit(): - local.append(f"cu{parts[0]}{parts[1]}") - if local: - return f"{base_version}+{'.'.join(local)}" - return base_version + cu = cuda.split(".") + if len(cu) >= 2 and cu[0].isdigit() and cu[1].isdigit(): + parts.append(f"cu{cu[0]}{cu[1]}") + if not parts: + return None + label = ".".join(parts) + if len(label) > 16 or not re.fullmatch(r"[a-z0-9._]+", label): + print( + f"{_RED}=== Variant label {label!r} violates PEP 817; skipping{_RESET}", + file=sys.stderr, + ) + return None + return label -def _repair_wheel_with_auditwheel(whl_dir, dest_dir): - """Upgrade a linux_ wheel to manylinux_2_X_. - - Ports the pattern established for tritonclient in TRI-286: - 1. auditwheel repair — auto-discovers the minimum manylinux tag - by inspecting glibc symbol requirements of the embedded .so. - 2. python -m wheel tags fallback — used when auditwheel reports - "no ELF" (the wheel has no native extension, e.g. a downstream - build disabled bindings). Mirrors the documented fallback. - 3. No-op with warning — when auditwheel is not installed in the - build image, keep the linux_ wheel as-is so the build - does not regress. +def _normalize_to_highest_manylinux(dist_dir): + """Collapse compressed manylinux tag sets to the highest version. + + auditwheel may emit a wheel whose PEP 425 platform-tag component + contains multiple manylinux entries joined by `.`, e.g. + `manylinux_2_27_x86_64.manylinux_2_28_x86_64`. Per project policy + (TRI-1118), keep only the highest version -- the strictest glibc + baseline. + + No-op for wheels already carrying a single platform tag. Other + non-manylinux entries in the compressed set are preserved. """ - if shutil.which("auditwheel") is None: + manylinux_re = re.compile(r"^manylinux_(\d+)_(\d+)_(.+)$") + for fname in os.listdir(dist_dir): + if not fname.endswith(".whl"): + continue + parts = fname[:-4].split("-") + if len(parts) < 5: + continue + plat = parts[-1] + if "." not in plat: + continue + tags = plat.split(".") + manylinux_tags = [] + other_tags = [] + for t in tags: + m = manylinux_re.match(t) + if m: + manylinux_tags.append(((int(m.group(1)), int(m.group(2))), t)) + else: + other_tags.append(t) + if len(manylinux_tags) <= 1 and not other_tags: + continue + if not manylinux_tags: + continue + manylinux_tags.sort() + highest = manylinux_tags[-1][1] + new_plat = ".".join([highest] + other_tags) if other_tags else highest + if new_plat == plat: + continue + wheel_path = os.path.join(dist_dir, fname) print( - "=== WARNING: auditwheel not found on PATH; keeping linux_ " - "wheel as-is. Install auditwheel in the build image to produce " - "PyPI-acceptable manylinux_2_X_ wheels.", + f"{_CYAN}=== Compressed platform tag in {fname!r}: " + f"{plat!r} -> {new_plat!r} (highest manylinux){_RESET}", file=sys.stderr, ) - shutil.copytree(os.path.join(whl_dir, "dist"), dest_dir, dirs_exist_ok=True) - return + r = subprocess.run( + [ + "python3", + "-m", + "wheel", + "tags", + "--platform-tag", + new_plat, + "--remove", + wheel_path, + ] + ) + fail_if(r.returncode != 0, "wheel tags normalization failed") + + +def _wheel_has_so(wheel_path): + """True if the wheel zip contains a native shared library. + Detects both unversioned (`libfoo.so`) and versioned (`libfoo.so.1.2`) + SONAMEs via filename inspection -- matches what auditwheel and pip + both use to classify wheels. + """ + with zipfile.ZipFile(wheel_path) as zf: + for name in zf.namelist(): + base = os.path.basename(name) + if base.endswith(".so") or ".so." in base: + return True + return False + + +def _repair_wheel_with_auditwheel(whl_dir, dest_dir): + """Apply the correct PEP 425 platform-compatibility tag to each wheel. + + Routing rules (per the relevant PEPs): + - Has native `.so` -> PEP 513 / PEP 599 / PEP 600 `manylinux___` + via `auditwheel repair`. auditwheel inspects the .so's glibc + symbol requirements and picks the lowest manylinux policy that + covers them, then bundles any non-allowlisted dynamic deps. + Original linux_ wheel is removed on success. + - No native `.so` -> PEP 425 pure-Python tag `py3-none-any`. + The manylinux platform tag is OMITTED -- claiming manylinux on + a wheel with no glibc-bound code would be a false compatibility + promise. + + Notes: + - PEP 656 musllinux is not produced here (build containers are + glibc-based; `auditwheel-musl` would be required on musl distros). + - PEP 440 version normalization happens upstream in main(), via + the dev-counter rewrite, before this function runs. + - If a wheel has a `.so` but `auditwheel` is missing from PATH, + the linux_ wheel is kept as-is and a warning is logged + rather than mis-tagging it as manylinux. + """ dist_dir = os.path.join(whl_dir, "dist") wheels = [ os.path.join(dist_dir, w) for w in os.listdir(dist_dir) if w.endswith(".whl") @@ -157,37 +241,72 @@ def _repair_wheel_with_auditwheel(whl_dir, dest_dir): fail_if(not wheels, "no wheel produced by the build") for wheel_path in wheels: - print(f"=== Running auditwheel repair on {wheel_path}") - r = subprocess.run( - ["auditwheel", "repair", wheel_path, "--wheel-dir", dest_dir], - capture_output=True, - text=True, - ) - if r.returncode != 0 and "no ELF" in r.stderr: - arch = os.uname().machine - manylinux_tag = f"manylinux_2_28_{arch}" + fname = os.path.basename(wheel_path) + # Skip wheels that already carry a manylinux/musllinux platform + # tag. Re-running auditwheel on an already-repaired wheel produces + # a compressed PEP 425 tag set + # (e.g. manylinux_2_27_x86_64.manylinux_2_28_x86_64) -- valid but + # noisy. This guards against CMake invoking this custom command + # twice (build + install phases) and finding stale wheels in dist/. + if "manylinux" in fname or "musllinux" in fname: + print( + f"{_CYAN}=== Skipping already-tagged wheel: {fname}{_RESET}", + file=sys.stderr, + ) + continue + if _wheel_has_so(wheel_path): + if shutil.which("auditwheel") is None: + print( + f"{_RED}=== WARNING: native .so found in " + f"{os.path.basename(wheel_path)} but auditwheel not on " + f"PATH; keeping linux_ wheel as-is. Install " + f"auditwheel in the build image to produce " + f"PyPI-acceptable manylinux wheels (PEP 513/599/600).{_RESET}", + file=sys.stderr, + ) + continue + print( + f"{_CYAN}=== Native extension in {os.path.basename(wheel_path)}: " + f"auditwheel repair -> PEP 513/599/600 manylinux{_RESET}", + file=sys.stderr, + ) + r = subprocess.run( + ["auditwheel", "repair", wheel_path, "--wheel-dir", dist_dir], + capture_output=True, + text=True, + ) + if r.returncode != 0: + sys.stderr.write(r.stderr) + fail_if(True, "auditwheel repair failed") + os.remove(wheel_path) + else: print( - f"=== Pure-Python wheel detected; falling back to wheel tags " - f"({manylinux_tag})" + f"{_CYAN}=== No native extension in " + f"{os.path.basename(wheel_path)}: retagging as PEP 425 " + f"pure-Python (py3-none-any); manylinux tag omitted{_RESET}", + file=sys.stderr, ) - copied = os.path.join(dest_dir, os.path.basename(wheel_path)) - shutil.copy(wheel_path, copied) - r2 = subprocess.run( + r = subprocess.run( [ "python3", "-m", "wheel", "tags", + "--python-tag", + "py3", + "--abi-tag", + "none", "--platform-tag", - manylinux_tag, + "any", "--remove", - copied, + wheel_path, ] ) - fail_if(r2.returncode != 0, "wheel tags fallback failed") - elif r.returncode != 0: - sys.stderr.write(r.stderr) - fail_if(True, "auditwheel repair failed") + fail_if(r.returncode != 0, "wheel tags retag failed for pure-Python wheel") + + # Post-process: if any resulting wheel carries a compressed manylinux + # tag set, collapse it to the highest version (project policy). + _normalize_to_highest_manylinux(dist_dir) def main(): @@ -202,12 +321,57 @@ def main(): required=True, help="Path to Triton Frontend Python binding.", ) + parser.add_argument( + "--release-version", + type=str, + required=False, + default=None, + help=( + "Base PEP 440 release version (e.g. '2.70.0'). Overrides the " + "TRITON_RELEASE_VERSION env var and the in-tree TRITON_VERSION file. " + "Precedence: --release-version > TRITON_RELEASE_VERSION > TRITON_VERSION file." + ), + ) FLAGS = parser.parse_args() - FLAGS.triton_version = None - with open("TRITON_VERSION", "r") as vfile: - FLAGS.triton_version = vfile.readline().strip() + # Base release version source — explicit precedence so CI can pin a + # release tag without editing the in-tree TRITON_VERSION file: + # 1. --release-version CLI flag + # 2. TRITON_RELEASE_VERSION env var + # 3. TRITON_VERSION file in CWD (legacy behaviour) + env_release_version = os.environ.get("TRITON_RELEASE_VERSION") + if FLAGS.release_version: + FLAGS.triton_version = FLAGS.release_version + base_source = "--release-version" + elif env_release_version: + FLAGS.triton_version = env_release_version + base_source = "TRITON_RELEASE_VERSION env" + else: + with open("TRITON_VERSION", "r") as vfile: + FLAGS.triton_version = vfile.readline().strip() + base_source = "TRITON_VERSION file" + print( + f"=== Wheel base version: {FLAGS.triton_version!r} (source: {base_source})", + file=sys.stderr, + ) + + # Replace the PEP 440 dev counter with CI_PIPELINE_ID when present, so + # each CI rebuild gets a monotonic, PyPI-uploadable, naturally-sortable + # version (e.g. 2.70.0.dev0 + CI_PIPELINE_ID=12345 -> 2.70.0.dev12345). + # Replaces the legacy PEP 427 build-tag scheme which PyPI rejects. + # Regex tolerates both 2.70.0.dev0 (canonical PEP 440) and 2.71.0dev + # (legacy in-tree shape with no period and no counter). + _pipeline = os.environ.get("CI_PIPELINE_ID", "") + _dev_m = re.match(r"^(\d+\.\d+\.\d+)\.?dev\d*$", FLAGS.triton_version) + if _dev_m and _pipeline.isdigit(): + _new = f"{_dev_m.group(1)}.dev{_pipeline}" + print( + f"{_CYAN}=== PEP 440 dev counter: {FLAGS.triton_version!r} -> " + f"{_new!r} (from CI_PIPELINE_ID={_pipeline}){_RESET}", + file=sys.stderr, + ) + FLAGS.triton_version = _new FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel") @@ -235,33 +399,33 @@ def main(): shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py")) os.chdir(FLAGS.whl_dir) + # Clean dist/ to prevent accumulating wheels from prior runs. CMake may + # invoke this custom command twice (build + install phases); without + # this, dist/ would end up with the linux_ wheel just produced + # AND the manylinux___ wheel left over from the previous + # run, and _repair_wheel_with_auditwheel would process both, producing + # wheels with compressed PEP 425 tag sets. + _dist = os.path.join(FLAGS.whl_dir, "dist") + if os.path.isdir(_dist): + shutil.rmtree(_dist) print("=== Building wheel") args = ["python3", "setup.py", "bdist_wheel"] - # PEP 427 build tag: lets two wheels of the same version coexist - # (e.g. reruns of the same CI pipeline). Sources, first non-empty - # and usable wins: - # CI_PIPELINE_ID - GitLab pipeline-scoped ID (preferred). - # NVIDIA_BUILD_ID - from build.py's --build-id flag. - # BUILD_NUMBER - generic CI systems. - # PEP 427 requires the build tag to start with a digit. - build_tag = ( - os.environ.get("CI_PIPELINE_ID") - or os.environ.get("NVIDIA_BUILD_ID") - or os.environ.get("BUILD_NUMBER") - ) + + # Release-semantic X.Y.Z -> PyPI-clean (no variant label). + # Anything else -> PEP 817 variant label. The pipeline id is already + # encoded as the PEP 440 .dev counter above, so no separate + # PEP 427 build tag is needed. + is_release = bool(re.match(r"^\d+\.\d+\.\d+$", FLAGS.triton_version)) print( - f"=== Wheel build-tag inputs: " - f"CI_PIPELINE_ID={os.environ.get('CI_PIPELINE_ID')!r} " - f"NVIDIA_BUILD_ID={os.environ.get('NVIDIA_BUILD_ID')!r} " - f"BUILD_NUMBER={os.environ.get('BUILD_NUMBER')!r} " - f"-> build-tag={build_tag!r}", + f"{_GREEN if is_release else _YELLOW}" + f"=== Version {FLAGS.triton_version!r} -> " + f"{'PEP 440 release (PyPI-clean)' if is_release else 'PEP 817 variant'}" + f"{_RESET}", file=sys.stderr, ) - if build_tag and build_tag != "" and build_tag[:1].isdigit(): - args += [f"--build-number={build_tag}"] wenv = os.environ.copy() - wenv["VERSION"] = _compose_version(FLAGS.triton_version) + wenv["VERSION"] = FLAGS.triton_version wenv["TRITON_PYBIND"] = PYBIND_LIB p = subprocess.Popen(args, env=wenv) p.wait() @@ -269,6 +433,25 @@ def main(): _repair_wheel_with_auditwheel(FLAGS.whl_dir, FLAGS.dest_dir) + if not is_release: + label = _compose_variant_label() + if label: + print( + f"{_CYAN}=== PEP 817 variant label: {label!r}{_RESET}", file=sys.stderr + ) + for fname in os.listdir(FLAGS.dest_dir): + if fname.endswith(".whl"): + os.rename( + os.path.join(FLAGS.dest_dir, fname), + os.path.join(FLAGS.dest_dir, fname[:-4] + f"-{label}.whl"), + ) + else: + print( + f"{_RED}=== PEP 817 variant: no nv/cu inputs detected; " + f"wheel emitted unlabeled{_RESET}", + file=sys.stderr, + ) + print(f"=== Output wheel file is in: {FLAGS.dest_dir}") touch(os.path.join(FLAGS.dest_dir, "stamp.whl"))