diff --git a/Dockerfile.sdk b/Dockerfile.sdk index 6697aff946..78e40924d3 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -41,7 +41,7 @@ ARG TRITON_ENABLE_GPU=ON ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG=1.5.8 ARG JAVA_BINDINGS_MAVEN_URL=https://archive.apache.org/dist/maven/maven-3/3.9.16/binaries/apache-maven-3.9.16-bin.tar.gz # DCGM version to install for Model Analyzer -ARG DCGM_VERSION=4.5.3-1 +ARG DCGM_VERSION=4.6.1-1 ARG NVIDIA_TRITON_SERVER_SDK_VERSION=unknown ARG NVIDIA_BUILD_ID=unknown diff --git a/build.py b/build.py index 6a36dbf05c..ac1c57f7c3 100755 --- a/build.py +++ b/build.py @@ -75,11 +75,10 @@ "release_version": "2.72.0dev", "triton_container_version": "26.08dev", "upstream_container_version": "26.07", - "ort_version": "1.27.0", - "ort_openvino_version": "2026.2.0", - "standalone_openvino_version": "2026.2.0", - "dcgm_version": "4.5.3-1", - "rhel_py_version": "3.12.3", + "ort_version": "1.28.0", + "ort_openvino_version": "2026.3.0", + "standalone_openvino_version": "2026.3.0", + "dcgm_version": "4.6.1-1", } CORE_BACKENDS = ["ensemble"] @@ -554,8 +553,6 @@ def backend_cmake_args(images, components, be, install_dir, library_paths): args = onnxruntime_cmake_args(images, library_paths) elif be == "openvino": args = openvino_cmake_args() - elif be == "python": - args = python_cmake_args() elif be == "dali": args = dali_cmake_args() elif be == "pytorch": @@ -614,18 +611,6 @@ def backend_cmake_args(images, components, be, install_dir, library_paths): return cargs -def python_cmake_args(): - cargs = [] - if target_platform() == "rhel": - cargs.append( - cmake_backend_arg( - "python", "PYBIND11_PYTHON_VERSION", "STRING", FLAGS.rhel_py_version - ) - ) - - return cargs - - def pytorch_cmake_args(images): if "pytorch" in images: image = images["pytorch"] @@ -925,9 +910,6 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): numactl-devel \\ openssl-devel \\ pkg-config \\ - python3-pip \\ - python3-scons \\ - python3-setuptools \\ rapidjson-devel \\ re2-devel \\ readline-devel \\ @@ -961,20 +943,21 @@ def create_dockerfile_buildbase_rhel(ddir, dockerfile_name, argmap): """.format( os.getenv("CCACHE_REMOTE_STORAGE") ) - # Requires openssl-devel to be installed first for pyenv build to be successful - df += change_default_python_version_rhel(FLAGS.rhel_py_version) + df += set_python_location_to_manylinux_for_rhel() + df += restore_embeddable_python_lib_rhel() df += """ RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ auditwheel \\ build \\ - wheel \\ - setuptools \\ + cmake==4.0.3 \\ docker \\ - virtualenv \\ patchelf==0.17.2 \\ - cmake==4.0.3 + scons \\ + setuptools \\ + virtualenv \\ + wheel """ df += f""" # Install boost version >= 1.78 for boost::span @@ -1335,9 +1318,10 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach libb64-devel \\ gperftools-devel \\ wget \\ - python3.12-pip \\ numactl-devel - +""" + df += set_python_location_to_manylinux_for_rhel() + df += """ RUN pip3 install patchelf==0.17.2 """ @@ -1412,8 +1396,7 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach openssl-devel \\ readline-devel """ - # Requires openssl-devel to be installed first for pyenv build to be successful - df += change_default_python_version_rhel(FLAGS.rhel_py_version) + df += set_python_location_to_manylinux_for_rhel() df += """ RUN pip3 install --upgrade pip \\ && pip3 install --upgrade \\ @@ -1554,30 +1537,30 @@ def add_cpu_libs_to_linux_dockerfile(backends, target_machine): return df -def change_default_python_version_rhel(version): - df = f""" -# The python library version available for install via 'yum install python3.X-devel' does not -# match the version of python inside the RHEL base container. This means that python packages -# installed within the container will not be picked up by the python backend stub process pybind -# bindings. It must instead must be installed via pyenv. -ENV PYENV_ROOT=/opt/pyenv_build -RUN curl https://pyenv.run | bash -ENV PATH="${{PYENV_ROOT}}/bin:$PATH" -RUN eval "$(pyenv init -)" -RUN CONFIGURE_OPTS=\"--with-openssl=/usr/lib64\" && pyenv install {version} \\ - && cp ${{PYENV_ROOT}}/versions/{version}/lib/libpython3* /usr/lib64/ - -# RHEL image has several python versions. It's important -# to set the correct version, otherwise, packages that are -# pip installed will not be found during testing. -ENV PYVER={version} PYTHONPATH=/opt/python/v -RUN ln -sf ${{PYENV_ROOT}}/versions/${{PYVER}}* ${{PYTHONPATH}} -ENV PYBIN=${{PYTHONPATH}}/bin -ENV PYTHON_BIN_PATH=${{PYBIN}}/python${{PYVER}} PATH=${{PYBIN}}:${{PATH}} +def set_python_location_to_manylinux_for_rhel(): + df = """ +ENV PATH="/opt/_internal/pipx/shared/bin:$PATH" """ return df +def restore_embeddable_python_lib_rhel(): + # The manylinux base container configures CPython with --disable-shared, + # so no libpython*.so is ever built, and its finalize.sh then compresses + # every libpython*.a into static-libs-for-embedding-only.tar.xz and drops + # the originals. triton_python_backend_stub links pybind11::embed and so + # needs that archive. Restore it in place -- the path it unpacks to is + # exactly the LIBDIR the interpreter's sysconfig reports, which is where + # pybind11's FindPythonLibsNew looks. + # + # This deliberately fails the build if the archive is absent: without it + # the stub link fails much later with an unresolvable -lpython. + return """ +RUN tar -xvf /opt/_internal/static-libs-for-embedding-only.tar.xz \\ + -C /opt/_internal +""" + + def create_build_dockerfiles( container_build_dir, images, backends, repoagents, caches, endpoints ): @@ -1987,17 +1970,6 @@ def backend_build( cmake_script.mkdir(os.path.join(install_dir, "backends")) cmake_script.rmdir(os.path.join(install_dir, "backends", be)) - # The python library version available for install via 'yum install python3.X-devel' does not - # match the version of python inside the RHEL base container. This means that python packages - # installed within the container will not be picked up by the python backend stub process pybind - # bindings. It must instead must be installed via pyenv. We package it here for better usability. - if target_platform() == "rhel" and be == "python": - major_minor_version = ".".join((FLAGS.rhel_py_version).split(".")[:2]) - version_matched_files = "/usr/lib64/libpython" + major_minor_version + "*" - cmake_script.cp( - version_matched_files, os.path.join(repo_install_dir, "backends", be) - ) - cmake_script.cpdir( os.path.join(repo_install_dir, "backends", be), os.path.join(install_dir, "backends"), @@ -2616,12 +2588,6 @@ def enable_all(): default=DEFAULT_TRITON_VERSION_MAP["dcgm_version"], help="This flag sets the DCGM version for Triton Inference Server to be built. Default: the latest supported version.", ) - parser.add_argument( - "--rhel-py-version", - required=False, - default=DEFAULT_TRITON_VERSION_MAP["rhel_py_version"], - help="This flag sets the Python version for RHEL platform of Triton Inference Server to be built. Default: the latest supported version.", - ) parser.add_argument( "--build-secret", action="append", diff --git a/compose.py b/compose.py index ef1342c0fe..b5b5e691e8 100755 --- a/compose.py +++ b/compose.py @@ -33,7 +33,7 @@ FLAGS = None -#### helper functions +# helper functions def log(msg, force=False): if force or not FLAGS.quiet: try: @@ -255,12 +255,12 @@ def create_argmap(images, skip_pull): vars = p_path.stdout log_verbose("inspect args: {}".format(vars)) - e0 = re.search("TRITON_SERVER_GPU_ENABLED=([\S]{1,}) ", vars) + e0 = re.search("TRITON_SERVER_GPU_ENABLED=([\\S]{1,}) ", vars) e1 = re.search("CUDA_VERSION", vars) gpu_enabled = False - if e0 != None: + if e0 is not None: gpu_enabled = e0.group(1) == "1" - elif e1 != None: + elif e1 is not None: gpu_enabled = True fail_if( gpu_enabled != enable_gpu, @@ -268,22 +268,22 @@ def create_argmap(images, skip_pull): "'TRITON_SERVER_GPU_ENABLED' as {} and you are composing container" "with 'TRITON_SERVER_GPU_ENABLED' as {}".format(gpu_enabled, enable_gpu), ) - e = re.search("TRITON_SERVER_VERSION=([\S]{6,}) ", vars) + e = re.search("TRITON_SERVER_VERSION=([\\S]{6,}) ", vars) version = "" if e is None else e.group(1) fail_if( len(version) == 0, "docker inspect to find triton server version failed, {}".format(p_path.stderr), ) - e = re.search("NVIDIA_TRITON_SERVER_VERSION=([\S]{5,}) ", vars) + e = re.search("NVIDIA_TRITON_SERVER_VERSION=([\\S]{5,}) ", vars) container_version = "" if e is None else e.group(1) fail_if( len(container_version) == 0, "docker inspect to find triton container version failed, {}".format(vars), ) - dcgm_ver = re.search("DCGM_VERSION=([\S]{4,}) ", vars) + dcgm_ver = re.search("DCGM_VERSION=([\\S]{4,}) ", vars) dcgm_version = "" if dcgm_ver is None: - dcgm_version = "4.5.3-1" + dcgm_version = "4.6.1-1" log( "WARNING: DCGM version not found from image, installing the earlierst version {}".format( dcgm_version diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh index b98ba607d6..ff12afd5bd 100755 --- a/qa/L0_backend_python/env/test.sh +++ b/qa/L0_backend_python/env/test.sh @@ -41,10 +41,9 @@ rm -rf *.tar.gz install_build_deps install_conda -# Test conda env without custom Python backend stub This environment should -# always use the default Python version shipped in the container. For Ubuntu -# 24.04 it is Python 3.12, for Ubuntu 22.04 is Python 3.10 and for Ubuntu 20.04 -# is 3.8. +# Test conda env without custom Python backend stub. This environment should +# always use the default Python version shipped in the container, which is +# 3.12 -- nothing below that is supported. path_to_conda_pack='$$TRITON_MODEL_DIRECTORY/python_3_12_environment.tar.gz' create_conda_env "3.12" "python-3-12" conda install -c conda-forge libstdcxx-ng=14 -y diff --git a/qa/L0_backend_python/examples/test.sh b/qa/L0_backend_python/examples/test.sh index 7f193a8d49..d17ac76983 100755 --- a/qa/L0_backend_python/examples/test.sh +++ b/qa/L0_backend_python/examples/test.sh @@ -42,33 +42,18 @@ pip3 uninstall -y numpy # NOTE: Using this subtest as a test case that involves using a python model with # numpy 2.X without changing the environments used in all the other test cases. if [ "$TEST_JETSON" == "0" ] && [[ ${TEST_WINDOWS} == 0 ]]; then - if [ ${PYTHON_ENV_VERSION} == "8" ]; then - # Python 3.8 does not support numpy 2.x, so installing numpy1.x - pip3 install "numpy<2" - pip3 install torch==2.0.0+cu117 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.15.0+cu117 - else - # Python 3.9 >= supports numpy 2.x. - pip3 install "numpy>=2" - pip3 install torch==2.5.0 torchvision==0.20.0 --index-url https://download.pytorch.org/whl/cu124 - fi + pip3 install "numpy>=2" + pip3 install torch==2.5.0 torchvision==0.20.0 --index-url https://download.pytorch.org/whl/cu124 else - if [ ${PYTHON_ENV_VERSION} == "8" ]; then - # Python 3.8 does not support numpy 2.x, so installing numpy1.x - pip3 install "numpy<2" - pip3 install torch==2.0.0 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.15.0 - else - # Python 3.9 >= supports numpy 2.x. - pip3 install "numpy>=2" - pip3 install torch==2.5.0 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.20.0 - fi + pip3 install "numpy>=2" + pip3 install torch==2.5.0 -f https://download.pytorch.org/whl/torch_stable.html torchvision==0.20.0 fi # Install `validators` for Model Instance Kind example pip3 install validators # Install JAX -# Jax has dropped the support for Python 3.8. See https://jax.readthedocs.io/en/latest/changelog.html -if [ "$TEST_JETSON" == "0" ] && [ ${PYTHON_ENV_VERSION} != "8" ]; then +if [ "$TEST_JETSON" == "0" ]; then pip install -U "jax[cuda12]" fi @@ -152,8 +137,7 @@ kill_server # JAX AddSub # JAX is not supported on Jetson -# Jax has dropped the support for Python 3.8. See https://jax.readthedocs.io/en/latest/changelog.html -if [ "$TEST_JETSON" == "0" ] && [ ${PYTHON_ENV_VERSION} != "8" ]; then +if [ "$TEST_JETSON" == "0" ]; then CLIENT_LOG="../examples_jax_client.log" mkdir -p models/jax/1/ cp examples/jax/model.py models/jax/1/model.py diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh deleted file mode 100755 index ce6be322ba..0000000000 --- a/qa/L0_backend_python/setup_python_enviroment.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -RET=0 -set -e -if [ ${PYTHON_ENV_VERSION} = "12" ]; then - echo No need to set up anything for default python3.${PYTHON_ENV_VERSION} - exit $RET -fi - -source common.sh -source ../common/util.sh - -TRITON_REPO_ORGANIZATION=${TRITON_REPO_ORGANIZATION:="http://github.com/triton-inference-server"} -BASE_SERVER_ARGS="--model-repository=${MODELDIR}/models --log-verbose=1 --disable-auto-complete-config" -PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG -SERVER_ARGS=$BASE_SERVER_ARGS -SERVER_LOG="./inference_server.log" -export PYTHON_ENV_VERSION=${PYTHON_ENV_VERSION:="12"} -RET=0 -EXPECTED_VERSION_STRINGS="" - -rm -fr ./models -rm -rf *.tar.gz -install_build_deps -install_conda - -# Test other python versions -conda update -n base -c defaults conda -y - -# Create a model with python 3.11 version -# Successful execution of the Python model indicates that the environment has -# been setup correctly. -if [ ${PYTHON_ENV_VERSION} = "11" ]; then - create_conda_env "3.11" "python-3-11" - conda install pytorch=2.8.0 -y - conda install -c conda-forge libstdcxx-ng=14 -y - conda install numpy=1.23.5 -y - EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and PyTorch version is 2.8.0" - create_python_backend_stub - conda-pack -o python3.11.tar.gz - path_to_conda_pack="$PWD/python-3-11" - mkdir -p $path_to_conda_pack - tar -xzf python3.11.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_11/1/ - cp ../python_models/python_version/config.pbtxt ./models/python_3_11 - (cd models/python_3_11 && \ - sed -i "s/^name:.*/name: \"python_3_11\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../python_models/python_version/model.py ./models/python_3_11/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_11 -fi -conda deactivate -rm -rf ./miniconda - -# test that -set +e -run_server -if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 -fi - -kill_server - -grep "$EXPECTED_VERSION_STRING" $SERVER_LOG -if [ $? -ne 0 ]; then - cat $SERVER_LOG - echo -e "\n***\n*** $EXPECTED_VERSION_STRING was not found in Triton logs. \n***" - RET=1 -fi -set -e - -echo "python environment 3.${PYTHON_ENV_VERSION}" -# copy the stub out to /opt/tritonserver/backends/python/triton_python_backend_stub -cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/triton_python_backend_stub -# Set up environment and stub for each test -apt-get update -qq && apt-get install -y software-properties-common -add-apt-repository ppa:deadsnakes/ppa -y -apt-get update && apt-get -y install \ - "python3.${PYTHON_ENV_VERSION}-dev" \ - "python3.${PYTHON_ENV_VERSION}-distutils" \ - libboost-dev -rm -f /usr/bin/python3 && \ -ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3 -pip3 install --upgrade requests numpy virtualenv protobuf -find /opt/tritonserver/qa/pkgs/ -maxdepth 1 -type f -name \ - "tritonclient-*-py3-none-any.whl" | xargs printf -- '%s[all]' | \ - xargs pip3 install --upgrade - -# Build triton-shm-monitor for the test -cd python_backend && rm -rf install build && mkdir build && cd build && \ - export CMAKE_POLICY_VERSION_MINIMUM=3.5 && \ - cmake -DCMAKE_INSTALL_PREFIX:PATH=$PWD/install \ - -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \ - -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ - -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \ - -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \ - make -j16 triton-shm-monitor install -cp $PWD/install/backends/python/triton_shm_monitor.cpython-* /opt/tritonserver/qa/common/. -set +e -exit $RET diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index a225c4504e..1e32fe96c1 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -84,14 +84,6 @@ if [ $? -ne 0 ]; then exit 1 fi -(bash -ex setup_python_enviroment.sh) - -python3 --version | grep "3.${PYTHON_ENV_VERSION}" > /dev/null -if [ $? -ne 0 ]; then - echo -e "Expecting Python version to be: Python 3.${PYTHON_ENV_VERSION} but actual version is $(python3 --version)" - exit 1 -fi - mkdir -p models/identity_fp32/1/ cp ../python_models/identity_fp32/model.py ./models/identity_fp32/1/model.py cp ../python_models/identity_fp32/config.pbtxt ./models/identity_fp32/config.pbtxt diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 3dd8847930..40d11443de 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -67,8 +67,14 @@ cd ${TRITON_MDLS_BASE_SCRIPT_DIR} log_message.status "define: default values" TRITON_VERSION=${TRITON_VERSION:=26.07} -ONNX_VERSION=1.20.1 -ONNX_OPSET=0 +ONNX_VERSION=1.22.0 # https://github.com/microsoft/onnxruntime/blob/v1.28.0/tools/ci_build/github/linux/python/requirements.txt#L15 +# Max opset ONNX Runtime accepts: the last *released* opset of the onnx that ORT +# was built against, i.e. one below that release's current opset. v1.28.0 pins +# onnx 1.22.0, whose opset 27 is still in development, so the ceiling is 26. +# Do NOT derive this from ONNX_VERSION -- 1.22.0 maps to 27, which fails to load. +# ORT's onnx pin: https://github.com/microsoft/onnxruntime/blob/v1.28.0/cmake/deps.txt +# As of 1.22.0 ONNX_OPSET is 26 - https://github.com/onnx/onnx/blob/v1.22.0/docs/Versioning.md#released-versions +ONNX_OPSET=26 OPENVINO_VERSION=2024.5.0 UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:22.04} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$TRITON_VERSION-py3} diff --git a/tools/build/build_presets.py b/tools/build/build_presets.py index 6177430469..0f0afc5da9 100644 --- a/tools/build/build_presets.py +++ b/tools/build/build_presets.py @@ -117,7 +117,6 @@ "TRITON_ENABLE_AZURE_STORAGE": "filesystem", "TRITON_ENABLE_ENSEMBLE": "backend", "TRITON_ENABLE_TENSORRT": "backend", - "PYBIND11_PYTHON_VERSION": "rhel_py_version", "TRITON_PYTORCH_DOCKER_IMAGE": "image", "TRITON_BUILD_CONTAINER": "image", "TRITON_BUILD_ONNXRUNTIME_VERSION": "ort_version",