From 4a23bb19ad5840114dcc72a25038a0e76ca85ef9 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 13 Oct 2025 14:00:27 -0700 Subject: [PATCH 01/21] update: Versions (#8459) --- Dockerfile.sdk | 2 +- README.md | 10 +++++----- TRITON_VERSION | 2 +- build.py | 6 +++--- deploy/aws/values.yaml | 2 +- deploy/fleetcommand/Chart.yaml | 2 +- deploy/fleetcommand/values.yaml | 6 +++--- deploy/gcp/values.yaml | 2 +- .../benchmark/perf-analyzer-script/triton_client.yaml | 2 +- .../server-deployer/build_and_push.sh | 6 +++--- .../server-deployer/chart/triton/Chart.yaml | 2 +- .../server-deployer/chart/triton/values.yaml | 6 +++--- .../server-deployer/data-test/schema.yaml | 2 +- deploy/gke-marketplace-app/server-deployer/schema.yaml | 4 ++-- deploy/gke-marketplace-app/trt-engine/README.md | 6 +++--- deploy/k8s-onprem/values.yaml | 2 +- deploy/oci/values.yaml | 2 +- docs/introduction/release_notes.md | 4 ++-- python/openai/README.md | 2 +- qa/common/gen_jetson_trt_models | 2 +- qa/common/gen_qa_custom_ops | 2 +- qa/common/gen_qa_model_repository | 2 +- 22 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Dockerfile.sdk b/Dockerfile.sdk index e919b3f50d..1804041101 100644 --- a/Dockerfile.sdk +++ b/Dockerfile.sdk @@ -29,7 +29,7 @@ # # Base image on the minimum Triton container -ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.09-py3-min +ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:25.10-py3-min ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server diff --git a/README.md b/README.md index 7be70a9ecc..cdd14ee53e 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ >[!WARNING] >You are currently on the `main` branch which tracks under-development progress ->towards the next release. The current release is version [2.61.0](https://github.com/triton-inference-server/server/releases/latest) ->and corresponds to the 25.09 container release on NVIDIA GPU Cloud (NGC). +>towards the next release. The current release is version [2.62.0](https://github.com/triton-inference-server/server/releases/latest) +>and corresponds to the 25.10 container release on NVIDIA GPU Cloud (NGC). # Triton Inference Server @@ -90,16 +90,16 @@ Inference Server with the ```bash # Step 1: Create the example model repository -git clone -b r25.09 https://github.com/triton-inference-server/server.git +git clone -b r25.10 https://github.com/triton-inference-server/server.git cd server/docs/examples ./fetch_models.sh # Step 2: Launch triton from the NGC Triton container -docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.09-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx +docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:25.10-py3 tritonserver --model-repository=/models --model-control-mode explicit --load-model densenet_onnx # Step 3: Sending an Inference Request # In a separate console, launch the image_client example from the NGC Triton SDK container -docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.09-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg +docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:25.10-py3-sdk /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg # Inference should return the following Image '/workspace/images/mug.jpg': diff --git a/TRITON_VERSION b/TRITON_VERSION index d9a000c4ed..995366c63c 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.62.0dev +2.62.0 diff --git a/build.py b/build.py index 4044aa599d..d02cf14200 100755 --- a/build.py +++ b/build.py @@ -71,9 +71,9 @@ # DEFAULT_TRITON_VERSION_MAP = { - "release_version": "2.62.0dev", - "triton_container_version": "25.10dev", - "upstream_container_version": "25.09", + "release_version": "2.62.0", + "triton_container_version": "25.10", + "upstream_container_version": "25.10", "ort_version": "1.23.1", "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", diff --git a/deploy/aws/values.yaml b/deploy/aws/values.yaml index 27bec3dd10..7c782b1ed9 100644 --- a/deploy/aws/values.yaml +++ b/deploy/aws/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/fleetcommand/Chart.yaml b/deploy/fleetcommand/Chart.yaml index 8b0eed9bae..2cbc1cbb34 100644 --- a/deploy/fleetcommand/Chart.yaml +++ b/deploy/fleetcommand/Chart.yaml @@ -26,7 +26,7 @@ apiVersion: v1 # appVersion is the Triton version; update when changing release -appVersion: 2.61.0" +appVersion: 2.62.0" description: Triton Inference Server (Fleet Command) name: triton-inference-server # version is the Chart version; update when changing anything in the chart diff --git a/deploy/fleetcommand/values.yaml b/deploy/fleetcommand/values.yaml index aa2e45e192..830407c9fe 100644 --- a/deploy/fleetcommand/values.yaml +++ b/deploy/fleetcommand/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent numGpus: 1 serverCommand: tritonserver @@ -47,13 +47,13 @@ image: # # To set model control mode, uncomment and configure below # TODO: Fix the following url, it is invalid - # See https://github.com/triton-inference-server/server/blob/r25.09/docs/user_guide/model_management.md + # See https://github.com/triton-inference-server/server/blob/r25.10/docs/user_guide/model_management.md # for more details #- --model-control-mode=explicit|poll|none # # Additional server args # - # see https://github.com/triton-inference-server/server/blob/r25.09/README.md + # see https://github.com/triton-inference-server/server/blob/r25.10/README.md # for more details service: diff --git a/deploy/gcp/values.yaml b/deploy/gcp/values.yaml index c4c2ff9951..b66268bce2 100644 --- a/deploy/gcp/values.yaml +++ b/deploy/gcp/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: gs://triton-inference-server-repository/model_repository numGpus: 1 diff --git a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml index 4b21697c29..d65a50862f 100644 --- a/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml +++ b/deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml @@ -33,7 +33,7 @@ metadata: namespace: default spec: containers: - - image: nvcr.io/nvidia/tritonserver:25.09-py3-sdk + - image: nvcr.io/nvidia/tritonserver:25.10-py3-sdk imagePullPolicy: Always name: nv-triton-client securityContext: diff --git a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh index b960b8ff94..9e2c400146 100755 --- a/deploy/gke-marketplace-app/server-deployer/build_and_push.sh +++ b/deploy/gke-marketplace-app/server-deployer/build_and_push.sh @@ -27,9 +27,9 @@ export REGISTRY=gcr.io/$(gcloud config get-value project | tr ':' '/') export APP_NAME=tritonserver -export MAJOR_VERSION=2.61 -export MINOR_VERSION=2.61.0 -export NGC_VERSION=25.09-py3 +export MAJOR_VERSION=2.62 +export MINOR_VERSION=2.62.0 +export NGC_VERSION=25.10-py3 docker pull nvcr.io/nvidia/$APP_NAME:$NGC_VERSION diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml index e86c2b6a21..87822f7953 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/Chart.yaml @@ -28,4 +28,4 @@ apiVersion: v1 appVersion: "2.60" description: Triton Inference Server name: triton-inference-server -version: 2.61.0 +version: 2.62.0 diff --git a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml index 9f87933228..3d8a3d14a2 100644 --- a/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml +++ b/deploy/gke-marketplace-app/server-deployer/chart/triton/values.yaml @@ -31,14 +31,14 @@ maxReplicaCount: 3 tritonProtocol: HTTP # HPA GPU utilization autoscaling target HPATargetAverageValue: 85 -modelRepositoryPath: gs://triton_sample_models/25.09 -publishedVersion: '2.61.0' +modelRepositoryPath: gs://triton_sample_models/25.10 +publishedVersion: '2.62.0' gcpMarketplace: true image: registry: gcr.io repository: nvidia-ngc-public/tritonserver - tag: 25.09-py3 + tag: 25.10-py3 pullPolicy: IfNotPresent # modify the model repository here to match your GCP storage bucket numGpus: 1 diff --git a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml index 107c3ce161..75af2cf0b7 100644 --- a/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/data-test/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.61.0' + publishedVersion: '2.62.0' publishedVersionMetadata: releaseNote: >- Initial release. diff --git a/deploy/gke-marketplace-app/server-deployer/schema.yaml b/deploy/gke-marketplace-app/server-deployer/schema.yaml index 4bc5fd29e6..1e61d21839 100644 --- a/deploy/gke-marketplace-app/server-deployer/schema.yaml +++ b/deploy/gke-marketplace-app/server-deployer/schema.yaml @@ -27,7 +27,7 @@ x-google-marketplace: schemaVersion: v2 applicationApiVersion: v1beta1 - publishedVersion: '2.61.0' + publishedVersion: '2.62.0' publishedVersionMetadata: releaseNote: >- Initial release. @@ -89,7 +89,7 @@ properties: modelRepositoryPath: type: string title: Bucket where models are stored. Please make sure the user/service account to create the GKE app has permission to this GCS bucket. Read Triton documentation on configs and formatting details, supporting TensorRT, TensorFlow, Pytorch, Onnx ... etc. - default: gs://triton_sample_models/25.09 + default: gs://triton_sample_models/25.10 image.ldPreloadPath: type: string title: Leave this empty by default. Triton allows users to create custom layers for backend such as TensorRT plugin, the compiled shared library must be provided via LD_PRELOAD environment variable. diff --git a/deploy/gke-marketplace-app/trt-engine/README.md b/deploy/gke-marketplace-app/trt-engine/README.md index f2118a1bc3..eb84e23fda 100644 --- a/deploy/gke-marketplace-app/trt-engine/README.md +++ b/deploy/gke-marketplace-app/trt-engine/README.md @@ -33,7 +33,7 @@ ``` docker run --gpus all -it --network host \ --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 \ - -v ~:/scripts nvcr.io/nvidia/tensorrt:25.09-py3 + -v ~:/scripts nvcr.io/nvidia/tensorrt:25.10-py3 pip install onnx six torch tf2onnx tensorflow @@ -57,7 +57,7 @@ mkdir -p engines python3 builder.py -m models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/model.ckpt -o engines/bert_large_int8_bs1_s128.engine -b 1 -s 128 -c models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/ -v models/fine-tuned/bert_tf_ckpt_large_qa_squad2_amp_128_v19.03.1/vocab.txt --int8 --fp16 --strict --calib-num 1 -iln -imh -gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.09/bert/1/model.plan +gsutil cp bert_large_int8_bs1_s128.engine gs://triton_sample_models/25.10/bert/1/model.plan ``` -For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.09/` should be updated accordingly with the correct version. +For each Triton upgrade, container version used to generate the model, and the model path in GCS `gs://triton_sample_models/25.10/` should be updated accordingly with the correct version. diff --git a/deploy/k8s-onprem/values.yaml b/deploy/k8s-onprem/values.yaml index 66f17e7c03..322ee46de2 100644 --- a/deploy/k8s-onprem/values.yaml +++ b/deploy/k8s-onprem/values.yaml @@ -30,7 +30,7 @@ tags: openshift: false image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryServer: < Replace with the IP Address of your file server > modelRepositoryPath: /srv/models diff --git a/deploy/oci/values.yaml b/deploy/oci/values.yaml index be4929a0f1..ba42839e92 100644 --- a/deploy/oci/values.yaml +++ b/deploy/oci/values.yaml @@ -27,7 +27,7 @@ replicaCount: 1 image: - imageName: nvcr.io/nvidia/tritonserver:25.09-py3 + imageName: nvcr.io/nvidia/tritonserver:25.10-py3 pullPolicy: IfNotPresent modelRepositoryPath: s3://https://.compat.objectstorage..oraclecloud.com:443/triton-inference-server-repository numGpus: 1 diff --git a/docs/introduction/release_notes.md b/docs/introduction/release_notes.md index 31fa2784c9..156ac4a7af 100644 --- a/docs/introduction/release_notes.md +++ b/docs/introduction/release_notes.md @@ -25,9 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> -# [Triton Inference Server Release 25.09](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-09.html#rel-25-09) +# [Triton Inference Server Release 25.10](https://docs.nvidia.com/deeplearning/triton-inference-server/release-notes/rel-25-10.html#rel-25-10) -The Triton Inference Server container image, release 25.09, is available +The Triton Inference Server container image, release 25.10, is available on [NGC](https://ngc.nvidia.com/catalog/containers/nvidia:tritonserver) and is open source on [GitHub](https://github.com/triton-inference-server/server). Release notes can diff --git a/python/openai/README.md b/python/openai/README.md index 5dda869bab..572373390f 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -51,7 +51,7 @@ docker run -it --net=host --gpus all --rm \ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ -e HF_TOKEN \ - nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 + nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 ``` 2. Launch the OpenAI-compatible Triton Inference Server: diff --git a/qa/common/gen_jetson_trt_models b/qa/common/gen_jetson_trt_models index 9b652a6977..693d233f91 100755 --- a/qa/common/gen_jetson_trt_models +++ b/qa/common/gen_jetson_trt_models @@ -34,7 +34,7 @@ # Make all generated files accessible outside of container umask 0000 # Set the version of the models -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} # Set the CUDA device to use CUDA_DEVICE=${RUNNER_ID:=0} # Set TensorRT image diff --git a/qa/common/gen_qa_custom_ops b/qa/common/gen_qa_custom_ops index 1d7f6db05a..a196fb3f8e 100755 --- a/qa/common/gen_qa_custom_ops +++ b/qa/common/gen_qa_custom_ops @@ -37,7 +37,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} NVIDIA_UPSTREAM_VERSION=${NVIDIA_UPSTREAM_VERSION:=$TRITON_VERSION} PYTORCH_IMAGE=${PYTORCH_IMAGE:=nvcr.io/nvidia/pytorch:$NVIDIA_UPSTREAM_VERSION-py3} UBUNTU_IMAGE=${UBUNTU_IMAGE:=ubuntu:24.04} diff --git a/qa/common/gen_qa_model_repository b/qa/common/gen_qa_model_repository index 9ac88218e5..753f6a672d 100755 --- a/qa/common/gen_qa_model_repository +++ b/qa/common/gen_qa_model_repository @@ -48,7 +48,7 @@ ## ############################################################################ -TRITON_VERSION=${TRITON_VERSION:=25.09} +TRITON_VERSION=${TRITON_VERSION:=25.10} # ONNX. Use ONNX_OPSET 0 to use the default for ONNX version ONNX_VERSION=1.16.1 From 2cfabeac5628108ddc4cf9fe2caafd93a6de12b0 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Thu, 16 Oct 2025 08:50:10 -0700 Subject: [PATCH 02/21] Add libnvshmem installation (#8460) --- build.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index d02cf14200..c5dc3b3920 100755 --- a/build.py +++ b/build.py @@ -78,7 +78,7 @@ "ort_openvino_version": "2025.3.0", "standalone_openvino_version": "2025.3.0", "dcgm_version": "4.4.0-1", - "vllm_version": "0.10.1.1", + "vllm_version": "0.10.2", "rhel_py_version": "3.12.3", } @@ -660,6 +660,10 @@ def pytorch_cmake_args(images): cargs.append( cmake_backend_enable("pytorch", "TRITON_ENABLE_NVTX", FLAGS.enable_nvtx) ) + if target_platform() == "igpu": + cargs.append( + cmake_backend_enable("pytorch", "TRITON_PYTORCH_NVSHMEM", False) + ) return cargs @@ -1514,6 +1518,22 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ + if target_platform() not in ["igpu", "windows", "rhel"]: + repo_arch = "sbsa" if target_machine == "aarch64" else "x86_64" + df += f""" +RUN curl -o /tmp/cuda-keyring.deb \\ + https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/{repo_arch}/cuda-keyring_1.1-1_all.deb \\ + && apt install /tmp/cuda-keyring.deb \\ + && rm /tmp/cuda-keyring.deb \\ + && apt update -qq \\ + && apt install --yes --no-install-recommends libnvshmem3-cuda-13 \\ + && rm -rf /var/lib/apt/lists/* \\ + && dpkg -L libnvshmem3-cuda-13 | grep libnvshmem_host.so | sed -e 's/libnvshmem_host.*//g' | sort -u > /etc/ld.so.conf.d/libnvshmem3-cuda-13.conf \\ + && ldconfig +""".format( + repo_arch=repo_arch + ) + df += """ WORKDIR /opt/tritonserver RUN rm -fr /opt/tritonserver/* From ea3c68784e5a8dccf51b65a716e75a449fb52082 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 20 Oct 2025 11:30:29 -0700 Subject: [PATCH 03/21] fix: Add environment variable defaults to BACKENDS (#8465) Co-authored-by: Dmitry Tokarev --- qa/L0_shared_memory/shared_memory_test.py | 8 +++++++- qa/L0_shared_memory/test.sh | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/qa/L0_shared_memory/shared_memory_test.py b/qa/L0_shared_memory/shared_memory_test.py index 6f8ba94337..5cb23d9869 100755 --- a/qa/L0_shared_memory/shared_memory_test.py +++ b/qa/L0_shared_memory/shared_memory_test.py @@ -301,7 +301,13 @@ def test_too_big_shm(self): def test_large_shm_register_offset(self): # Test for out of bounds read vulnerability when registering system shared memory with large offset - for platform in ["python", "onnx", "libtorch", "plan", "openvino"]: + + platforms = ( + ["python", "onnx", "libtorch", "plan", "openvino"] + if os.environ.get("BACKENDS") is None + else os.environ.get("BACKENDS").split() + ) + for platform in platforms: model_name = f"{platform}_int32_int32_int32" # Test for large offset diff --git a/qa/L0_shared_memory/test.sh b/qa/L0_shared_memory/test.sh index 3f5758bf36..e33c9e3756 100755 --- a/qa/L0_shared_memory/test.sh +++ b/qa/L0_shared_memory/test.sh @@ -160,7 +160,7 @@ done # Test large system shared memory offset rm -rf models/* # prepare add_sub model of various backends -BACKENDS="python onnx libtorch plan openvino" +BACKENDS=${BACKENDS:-"python onnx libtorch plan openvino"} for backend in ${BACKENDS} ; do model="${backend}_int32_int32_int32" model_dir="models/${model}" From 46fe7a8a1978ebd9b08049d3667d252b834b8f08 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Mon, 27 Oct 2025 10:23:25 -0700 Subject: [PATCH 04/21] Removing unused values for TensorRT-LLM container build (#8472) --- build.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/build.py b/build.py index c5dc3b3920..97679c37f9 100755 --- a/build.py +++ b/build.py @@ -1283,10 +1283,9 @@ def create_dockerfile_linux( pip3 uninstall -y setuptools ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH -# There are some ucc issues when spawning mpi processes with ompi v4.1.7a1. -# Downgrade to ompi v4.1.5rc2 to avoid the issue. -RUN rm -fr /opt/hpcx/ompi -COPY --from=nvcr.io/nvidia/tritonserver:24.02-py3-min /opt/hpcx/ompi /opt/hpcx/ompi +RUN apt-get update && apt-get install -y --no-install-recommends \ + libprotobuf-dev \ + && rm -rf /var/lib/apt/lists/* """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) @@ -1465,14 +1464,6 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach virtualenv \\ && rm -rf /var/lib/apt/lists/* """ - if "tensorrtllm" in backends: - df += """ -# Updating the openssh-client to fix for the CVE-2024-6387. This can be removed when trtllm uses a later CUDA container(12.5 or later) -RUN apt-get update \\ - && apt-get install -y --no-install-recommends \\ - openssh-client \\ - && rm -rf /var/lib/apt/lists/* - """ if "vllm" in backends: df += f""" From 31ebf6adc5ddae2abf0ff18c50cae9e121ac62a7 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Tue, 28 Oct 2025 14:23:28 -0700 Subject: [PATCH 05/21] rollback: Reverting TensorRT-LLM Backend changes. (#8478) --- build.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.py b/build.py index 97679c37f9..d41b599900 100755 --- a/build.py +++ b/build.py @@ -1282,10 +1282,6 @@ def create_dockerfile_linux( pip3 install --no-cache-dir grpcio-tools==1.64.0 && \\ pip3 uninstall -y setuptools ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib/:/opt/tritonserver/backends/tensorrtllm:$LD_LIBRARY_PATH - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libprotobuf-dev \ - && rm -rf /var/lib/apt/lists/* """ with open(os.path.join(ddir, dockerfile_name), "w") as dfile: dfile.write(df) From 33b6a149e53ecfeb6e514a099ee0f1d137705c7b Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 29 Oct 2025 22:43:14 +0530 Subject: [PATCH 06/21] ci: Fix TensorRT engine build error for vision models (#8479) --- qa/common/gen_qa_image_models.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/qa/common/gen_qa_image_models.py b/qa/common/gen_qa_image_models.py index a32ba92f1b..970d08662c 100755 --- a/qa/common/gen_qa_image_models.py +++ b/qa/common/gen_qa_image_models.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -99,7 +99,8 @@ def export_vgg19(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -107,6 +108,7 @@ def export_vgg19(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"VGG19 model exported to: {model_path}") @@ -129,7 +131,8 @@ def export_resnet152(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -137,6 +140,7 @@ def export_resnet152(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"ResNet-152 model exported to: {model_path}") @@ -159,7 +163,8 @@ def export_resnet50(models_dir, model_name="model.onnx"): model.eval() dummy_input = torch.randn(1, 3, 224, 224) # (batch, channels, height, width) - # Export the model to ONNX format + # Use legacy TorchScript-based ONNX export + # TODO: Update to use new torch.export-based ONNX exporter (default dynamo=True) torch.onnx.export( model, dummy_input, @@ -167,6 +172,7 @@ def export_resnet50(models_dir, model_name="model.onnx"): input_names=["input"], output_names=["output"], dynamic_axes={"input": {0: "batch_size"}, "output": {0: "batch_size"}}, + dynamo=False, ) print(f"ResNet-50 model exported to: {model_path}") From da6d4bb68d66c862409245fbdd8f7e1ac23dc7ee Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 10:45:15 -0700 Subject: [PATCH 07/21] Remove additional installation (#8480) --- build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.py b/build.py index d41b599900..63cc987682 100755 --- a/build.py +++ b/build.py @@ -1505,7 +1505,10 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach ENV PYTHONPATH=/opt/tritonserver/backends/dali/wheel/dali:$PYTHONPATH """ - if target_platform() not in ["igpu", "windows", "rhel"]: + if ( + target_platform() not in ["igpu", "windows", "rhel"] + and "tensorrtllm" not in backends + ): repo_arch = "sbsa" if target_machine == "aarch64" else "x86_64" df += f""" RUN curl -o /tmp/cuda-keyring.deb \\ From 2961724ddc82b8c55afc9b5bac4532598e372295 Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 15:52:06 -0700 Subject: [PATCH 08/21] docs: Update release related versions and values (#8481) --- docs/introduction/compatibility.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/introduction/compatibility.md b/docs/introduction/compatibility.md index 1e1dbb5e3c..c90899361b 100644 --- a/docs/introduction/compatibility.md +++ b/docs/introduction/compatibility.md @@ -38,6 +38,7 @@ | Triton release version | NGC Tag | Python version | Torch version | TensorRT version | TensorRT-LLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 25.10 | nvcr.io/nvidia/tritonserver:25.10-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.21 GB | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.6 | 10.11.0.33 | 1.0.0 | 12.9.1.010 | 575.57.08 | 16.25 GB | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-trtllm-python-py3 | Python 3.12.3 | 2.8.0a0+5228986c39.nv25.5 | 10.11.0.33 | 0.21.0 | 12.9.0.043 | 575.51.03 | 20.49 GB | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-trtllm-python-py3 | Python 3.12.3 | 2.7.0a0+79aa17489c.nv25.4 | 10.10.0.31 | 0.20.0 | 12.9.0.036 | 575.51.03 | 18.3G | @@ -61,6 +62,7 @@ | Triton release version | NGC Tag | Python version | vLLM version | CUDA version | CUDA Driver version | Size | | --- | --- | --- | --- | --- | --- | --- | +| 25.10 | nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 | Python 3.12.3 | 0.10.2+9dd9ca32.nv25.10.cu130 | 13.0.2.006 | 580.95.05 | 8.34G | | 25.09 | nvcr.io/nvidia/tritonserver:25.09-vllm-python-py3 | Python 3.12.3 | 0.10.1.1+381074ae.nv25.9.cu130 | 13.0.1.012 | 580.82.07 | 7.78G | | 25.08 | nvcr.io/nvidia/tritonserver:25.08-vllm-python-py3 | Python 3.12.3 | 0.9.2+4ef1e343.nv25.8.post1.cu130 | 13.0.1.012 | 580.82.07 | 8.1G | | 25.07 | nvcr.io/nvidia/tritonserver:25.07-vllm-python-py3 | Python 3.12.3 | 0.9.0rc1+1958ee56.nv25.6.cu129 | 12.9.0.043 | 575.51.03 | 10G | @@ -84,6 +86,7 @@ | Triton release version | ONNX Runtime | | --- | --- | +| 25.10 | 1.23.1 | | 25.09 | 1.23.0 | | 25.08 | 1.23.0+1d1712fdaf | | 25.07 | 1.22.0 | From 9878606eee34982c5eaff9ecfab84258afe06133 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 29 Oct 2025 17:54:36 -0700 Subject: [PATCH 09/21] Add embeddings endpoint --- python/openai/README.md | 2 + .../openai/openai_frontend/engine/engine.py | 10 +- .../openai_frontend/engine/triton_engine.py | 143 ++++- .../openai_frontend/engine/utils/triton.py | 66 ++- .../fastapi/middleware/api_restriction.py | 6 +- .../frontend/fastapi/routers/embeddings.py | 50 ++ .../frontend/fastapi_frontend.py | 9 +- .../openai/openai_frontend/schemas/openai.py | 83 ++- python/openai/tests/conftest.py | 121 ++-- python/openai/tests/test_embeddings.py | 553 ++++++++++++++++++ 10 files changed, 953 insertions(+), 90 deletions(-) create mode 100644 python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py create mode 100644 python/openai/tests/test_embeddings.py diff --git a/python/openai/README.md b/python/openai/README.md index 5dda869bab..39aa8f9666 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -655,6 +655,8 @@ Use the `--openai-restricted-api` command-line argument to configure endpoint re - **inference**: Chat completions and text completions endpoints - `POST /v1/chat/completions` - `POST /v1/completions` + - **embedding**: Embedding endpoint + - `POST /v1/embeddings` - **model-repository**: Model listing and information endpoints - `GET /v1/models` - `GET /v1/models/{model_name}` diff --git a/python/openai/openai_frontend/engine/engine.py b/python/openai/openai_frontend/engine/engine.py index 9c90dec25e..527ef9542d 100644 --- a/python/openai/openai_frontend/engine/engine.py +++ b/python/openai/openai_frontend/engine/engine.py @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,6 +34,8 @@ CreateChatCompletionResponse, CreateCompletionRequest, CreateCompletionResponse, + CreateEmbeddingRequest, + CreateEmbeddingResponse, Model, ) @@ -92,3 +94,9 @@ def completion( If request.stream is False, this returns a CreateCompletionResponse. """ pass + + def embed(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: + """ + Returns a CreateEmbeddingResponse. + """ + pass diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 499cc623e7..3f8ca8cff6 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -27,6 +27,7 @@ from __future__ import annotations +import base64 import json import time import uuid @@ -38,18 +39,24 @@ Callable, Dict, List, + Literal, Optional, Tuple, + Union, ) +import numpy as np import tritonserver from engine.engine import LLMEngine from engine.utils.chat import load_chat_template, parse_chat_messages from engine.utils.tokenizer import get_tokenizer from engine.utils.tool_call_parsers import ToolCallParser, ToolParserManager from engine.utils.triton import ( - _create_trtllm_inference_request, - _create_vllm_inference_request, + RequestKind, + _create_trtllm_embedding_request, + _create_trtllm_generate_request, + _create_vllm_embedding_request, + _create_vllm_generate_request, _get_output, _get_usage_from_response, _get_vllm_lora_names, @@ -73,6 +80,9 @@ CreateChatCompletionStreamResponse, CreateCompletionRequest, CreateCompletionResponse, + CreateEmbeddingRequest, + CreateEmbeddingResponse, + EmbeddingObject, FinishReason, Function1, Function2, @@ -97,7 +107,8 @@ class TritonModelMetadata: # Time that model was loaded by Triton create_time: int # Conversion format between OpenAI and Triton requests - request_converter: Callable + inference_request_converter: Callable + embedding_request_converter: Callable class TritonLLMEngine(LLMEngine): @@ -189,7 +200,7 @@ async def chat( # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.request_converter( + metadata.inference_request_converter( metadata.model, prompt, request, lora_name, self.default_max_tokens ) ) @@ -232,7 +243,9 @@ async def chat( backend=metadata.backend, ) - usage = _get_usage_from_response(response, metadata.backend) + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.GENERATE + ) return CreateChatCompletionResponse( id=request_id, @@ -311,7 +324,7 @@ async def completion( # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.request_converter( + metadata.inference_request_converter( metadata.model, request.prompt, request, @@ -334,7 +347,9 @@ async def completion( response = responses[0] text = _get_output(response) - usage = _get_usage_from_response(response, metadata.backend) + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.GENERATE + ) choice = Choice( finish_reason=FinishReason.stop, @@ -352,6 +367,53 @@ async def completion( usage=usage, ) + async def embed(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: + # Validate request and convert to Triton format + model_name, _ = self._get_model_and_lora_name(request.model) + metadata = self.model_metadata.get(model_name) + self._validate_embedding_request(request, metadata) + + # Convert to Triton request format and perform inference + responses = metadata.model.async_infer( + metadata.embedding_request_converter( + metadata.model, + request, + ) + ) + + # Response validation with decoupled models in mind + responses = [response async for response in responses] + _validate_triton_responses_non_streaming(responses) + response = responses[0] + + # Extract embedding from response (currently stored as JSON string in text_output) + embedding_json = _get_output(response) + embedding_list = json.loads(embedding_json) + + usage = _get_usage_from_response(response, metadata.backend, RequestKind.EMBED) + + embedding = self._get_embedding(embedding_list, request.encoding_format) + embedding_obj = EmbeddingObject( + embedding=embedding, index=0, object="embedding" + ) + + return CreateEmbeddingResponse( + object="list", + data=[embedding_obj], + model=request.model, + usage=usage, + ) + + @staticmethod + def _get_embedding( + embedding: List[float], encoding_format: Literal["float", "base64"] + ) -> Union[list[float], str]: + if encoding_format == "float": + return embedding + elif encoding_format == "base64": + embedding_bytes = np.array(embedding, dtype="float32").tobytes() + return base64.b64encode(embedding_bytes).decode("utf-8") + # TODO: This behavior should be tested further def _get_first_response_role( self, conversation: List[Dict], add_generation_prompt: bool, default_role: str @@ -362,18 +424,24 @@ def _get_first_response_role( return conversation[-1]["role"] # TODO: Expose explicit flag to catch edge cases - def _determine_request_converter(self, backend: str): + def _determine_request_converter(self, backend: str, request_type: RequestKind): # Allow manual override of backend request format if provided by user if self.backend: backend = self.backend # Request conversion from OpenAI format to backend-specific format if backend == "vllm": - return _create_vllm_inference_request + if request_type == RequestKind.GENERATE: + return _create_vllm_generate_request + else: + return _create_vllm_embedding_request # Use TRT-LLM format as default for everything else. This could be # an ensemble, a python or BLS model, a TRT-LLM backend model, etc. - return _create_trtllm_inference_request + if request_type == RequestKind.GENERATE: + return _create_trtllm_generate_request + else: + return _create_trtllm_embedding_request def _get_model_and_lora_name(self, request_model_name: str): if self.lora_separator is None or len(self.lora_separator) == 0: @@ -418,7 +486,12 @@ def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]: tokenizer=self.tokenizer, lora_names=lora_names, create_time=self.create_time, - request_converter=self._determine_request_converter(backend), + inference_request_converter=self._determine_request_converter( + backend, RequestKind.GENERATE + ), + embedding_request_converter=self._determine_request_converter( + backend, RequestKind.EMBED + ), ) model_metadata[name] = metadata @@ -671,8 +744,15 @@ def _validate_chat_request( if not metadata.backend: raise Exception("Unknown backend") - if not metadata.request_converter: - raise Exception(f"Unknown request format for model: {request.model}") + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) if ( metadata.lora_names is not None @@ -807,8 +887,15 @@ def _validate_completion_request( if not metadata.backend: raise Exception("Unknown backend") - if not metadata.request_converter: - raise Exception(f"Unknown request format for model: {request.model}") + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) if ( metadata.lora_names is not None @@ -853,6 +940,32 @@ def _validate_completion_request( "`stream_options.include_usage` is currently only supported for the vLLM backend" ) + def _validate_embedding_request( + self, + request: CreateEmbeddingRequest, + metadata: TritonModelMetadata, + ): + """ + Validates an embedding request to align with currently supported features. + """ + + # Reject missing internal information needed to do inference + if not metadata: + raise Exception(f"Unknown model: {request.model}") + + if not metadata.backend: + raise Exception("Unknown backend") + + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) + def _should_stream_with_auto_tool_parsing( self, request: CreateChatCompletionRequest ): diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 636e58435d..35aa75fea0 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -28,6 +28,7 @@ import os import re from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Iterable, List, Optional, Union @@ -40,10 +41,17 @@ CompletionUsage, CreateChatCompletionRequest, CreateCompletionRequest, + CreateEmbeddingRequest, + EmbeddingUsage, ) -def _create_vllm_inference_request( +class RequestKind(Enum): + GENERATE = "generate" + EMBED = "embed" + + +def _create_vllm_generate_request( model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest, @@ -128,7 +136,7 @@ def _create_vllm_inference_request( return model.create_request(inputs=inputs) -def _create_trtllm_inference_request( +def _create_trtllm_generate_request( model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest, @@ -183,6 +191,35 @@ def _create_trtllm_inference_request( return model.create_request(inputs=inputs) +def _create_vllm_embedding_request( + model, + request: CreateEmbeddingRequest, +): + inputs = {} + embedding_request = {} + embedding_request["input"] = request.input + + pooling_params = {} + dims = request.dimensions + if dims is not None: + pooling_params["dimensions"] = [dims] + embedding_request["pooling_params"] = pooling_params + + inputs["embedding_request"] = [json.dumps(embedding_request)] + inputs["return_num_input_tokens"] = np.bool_([True]) + inputs["return_num_output_tokens"] = np.bool_([True]) + return model.create_request(inputs=inputs) + + +def _create_trtllm_embedding_request( + model, + request: CreateEmbeddingRequest, +): + raise Exception( + "TRT-LLM backend and Python backend do not support embedding requests" + ) + + def _construct_string_from_pointer(pointer: int, size: int) -> str: """Constructs a Python string from a C pointer and size.""" @@ -236,7 +273,7 @@ class _StreamingUsageAccumulator: def update(self, response: tritonserver.InferenceResponse): """Extracts usage from a response and updates the token counts.""" - usage = _get_usage_from_response(response, self.backend) + usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATE) if usage: # The prompt_tokens is received with every chunk but should only be set once. if not self._prompt_tokens_set: @@ -262,7 +299,8 @@ def get_final_usage(self) -> Optional[CompletionUsage]: def _get_usage_from_response( response: tritonserver._api._response.InferenceResponse, backend: str, -) -> Optional[CompletionUsage]: + request_type: RequestKind, +) -> Optional[CompletionUsage | EmbeddingUsage]: """ Extracts token usage statistics from a Triton inference response. """ @@ -293,13 +331,19 @@ def _get_usage_from_response( ) completion_tokens = completion_tokens_ptr[0] - if prompt_tokens is not None and completion_tokens is not None: - total_tokens = prompt_tokens + completion_tokens - return CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + if prompt_tokens is not None: + if request_type == RequestKind.GENERATE and completion_tokens is not None: + total_tokens = prompt_tokens + completion_tokens + return CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + elif request_type == RequestKind.EMBED: + return EmbeddingUsage( + prompt_tokens=prompt_tokens, + total_tokens=prompt_tokens, + ) return None diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py index 9b73169ade..abe452a595 100644 --- a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py +++ b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py @@ -30,7 +30,11 @@ # Mapping of API to their corresponding HTTP endpoints ENDPOINT_MAPPING = { - "inference": ["POST /v1/chat/completions", "POST /v1/completions"], + "inference": [ + "POST /v1/chat/completions", + "POST /v1/completions", + "POST /v1/embeddings", + ], "model-repository": ["GET /v1/models"], "metrics": ["GET /metrics"], "health": ["GET /health/ready"], diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py new file mode 100644 index 0000000000..9da54b56cb --- /dev/null +++ b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py @@ -0,0 +1,50 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse +from schemas.openai import CreateEmbeddingRequest, CreateEmbeddingResponse + +router = APIRouter() + + +@router.post( + "/v1/embeddings", response_model=CreateEmbeddingResponse, tags=["Embeddings"] +) +async def create_embedding( + request: CreateEmbeddingRequest, raw_request: Request +) -> CreateEmbeddingResponse | StreamingResponse: + """ + Creates embedding for the provided input text. + """ + if not raw_request.app.engine: + raise HTTPException(status_code=500, detail="No attached inference engine") + + try: + response = await raw_request.app.engine.embed(request) + return response + except Exception as e: + raise HTTPException(status_code=400, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi_frontend.py b/python/openai/openai_frontend/frontend/fastapi_frontend.py index 2234d84208..e5fb01deae 100644 --- a/python/openai/openai_frontend/frontend/fastapi_frontend.py +++ b/python/openai/openai_frontend/frontend/fastapi_frontend.py @@ -34,7 +34,13 @@ APIRestrictionMiddleware, RestrictedFeatures, ) -from frontend.fastapi.routers import chat, completions, models, observability +from frontend.fastapi.routers import ( + chat, + completions, + embeddings, + models, + observability, +) from frontend.frontend import OpenAIFrontend @@ -97,6 +103,7 @@ def _create_app(self): app.include_router(models.router) app.include_router(completions.router) app.include_router(chat.router) + app.include_router(embeddings.router) # NOTE: For debugging purposes, should generally be restricted or removed self._add_cors_middleware(app) diff --git a/python/openai/openai_frontend/schemas/openai.py b/python/openai/openai_frontend/schemas/openai.py index f13296163d..81ff6e93b3 100644 --- a/python/openai/openai_frontend/schemas/openai.py +++ b/python/openai/openai_frontend/schemas/openai.py @@ -31,7 +31,7 @@ from __future__ import annotations from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel, confloat, conint @@ -601,10 +601,7 @@ class Model(BaseModel): owned_by: str = Field(..., description="The organization that owns the model.") -class CompletionUsage(BaseModel): - completion_tokens: int = Field( - ..., description="Number of tokens in the generated completion." - ) +class BaseUsage(BaseModel): prompt_tokens: int = Field(..., description="Number of tokens in the prompt.") total_tokens: int = Field( ..., @@ -612,6 +609,16 @@ class CompletionUsage(BaseModel): ) +class EmbeddingUsage(BaseUsage): + pass + + +class CompletionUsage(BaseUsage): + completion_tokens: int = Field( + ..., description="Number of tokens in the generated completion." + ) + + class Event(Enum): error = "error" @@ -940,3 +947,69 @@ class ObjectType: text_completion = Object1.text_completion chat_completion_chunk = Object4.chat_completion_chunk chat_completion = Object2.chat_completion + + +class EmbeddingObject(BaseModel): + model_config: ConfigDict = ConfigDict(extra="forbid") + + object: Literal["embedding"] = Field( + description="The object type, which is always 'embedding'.", + ) + embedding: Union[List[float], str] = Field( + ..., + description="The embedding vector, which is a list of floats or a base64-encoded string.", + ) + index: int = Field( + ..., + description="The index of the embedding in the list of embeddings.", + ) + + +class CreateEmbeddingRequest(BaseModel): + # Explicitly return errors for unknown fields. + model_config: ConfigDict = ConfigDict(extra="forbid") + + input: Union[str, List[int]] = Field( + ..., + description="Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.", + min_length=1, + examples=["The food was delicious and the waiter..."], + ) + model: Union[str, Model2] = Field( + ..., + description="ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API.", + examples=["text-embedding-ada-002"], + ) + dimensions: Optional[int] = Field( + None, + description="The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.", + ) + encoding_format: Optional[Literal["float", "base64"]] = Field( + "float", + description="The format to return the embeddings in.", + ) + user: Optional[str] = Field( + None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n", + examples=["user-1234"], + ) + + +class CreateEmbeddingResponse(BaseModel): + model_config: ConfigDict = ConfigDict(extra="forbid") + + object: Literal["list"] = Field( + description="The object type, which is always 'list'.", + ) + data: List[EmbeddingObject] = Field( + ..., + description="The list of embeddings.", + ) + model: Union[str, Model2] = Field( + ..., + description="The model used to generate the embeddings.", + ) + usage: Optional[EmbeddingUsage] = Field( + ..., + description="The usage for the request.", + ) diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index 50ba0de4ed..d05ce697c0 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -67,41 +67,80 @@ def infer_test_model_repository(backend, tool_call_parser): return model_repository -# TODO: Refactor away from global variables -TEST_MODEL = os.environ.get("TEST_MODEL") -TEST_BACKEND = os.environ.get("TEST_BACKEND") -TEST_MODEL_REPOSITORY = os.environ.get("TEST_MODEL_REPOSITORY") +### FIXTURES - Refactored from global variables ### -TEST_TOKENIZER = os.environ.get( - "TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct" -) -TEST_TOOL_CALL_PARSER = os.environ.get("TEST_TOOL_CALL_PARSER", "llama3") -TEST_PROMPT = "What is machine learning?" -TEST_MESSAGES = [{"role": "user", "content": TEST_PROMPT}] -if not TEST_BACKEND or not TEST_MODEL: - TEST_BACKEND, TEST_MODEL = infer_test_environment(TEST_TOOL_CALL_PARSER) +@pytest.fixture(scope="session") +def tool_call_parser(): + return os.environ.get("TEST_TOOL_CALL_PARSER", "llama3") -if not TEST_MODEL_REPOSITORY: - TEST_MODEL_REPOSITORY = infer_test_model_repository( - TEST_BACKEND, TEST_TOOL_CALL_PARSER - ) + +@pytest.fixture(scope="session") +def backend(tool_call_parser): + env_backend = os.environ.get("TEST_BACKEND") + env_model = os.environ.get("TEST_MODEL") + + if not env_backend or not env_model: + inferred_backend, _ = infer_test_environment(tool_call_parser) + return inferred_backend + return env_backend + + +@pytest.fixture(scope="session") +def model(backend, tool_call_parser): + env_model = os.environ.get("TEST_MODEL") + + if not env_model: + _, inferred_model = infer_test_environment(tool_call_parser) + return inferred_model + return env_model + + +@pytest.fixture(scope="session") +def model_repository(backend, tool_call_parser): + env_repo = os.environ.get("TEST_MODEL_REPOSITORY") + + if env_repo: + return env_repo + return infer_test_model_repository(backend, tool_call_parser) + + +@pytest.fixture(scope="session") +def tokenizer_model(): + return os.environ.get("TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct") + + +@pytest.fixture(scope="session") +def prompt(): + return "What is machine learning?" + + +@pytest.fixture(scope="session") +def messages(prompt): + return [{"role": "user", "content": prompt}] + + +@pytest.fixture(scope="session") +def input(prompt): + return prompt # NOTE: OpenAI client requires actual server running, and won't work # with the FastAPI TestClient. Run the server at module scope to run # only once for all the tests below. @pytest.fixture(scope="module") -def server(): +def server( + model_repository: str, tokenizer_model: str, backend: str, tool_call_parser: str +): args = [ "--model-repository", - TEST_MODEL_REPOSITORY, + model_repository, "--tokenizer", - TEST_TOKENIZER, + tokenizer_model, "--backend", - TEST_BACKEND, + backend, "--tool-call-parser", - TEST_TOOL_CALL_PARSER, + tool_call_parser, ] # TODO: Incorporate kserve frontend binding smoke tests to catch any # breakage with default values or slight cli arg variations @@ -117,47 +156,17 @@ def server(): # with arbitrary clients - you must use the TestClient returned to interact with # the "server" when "starting the server" via TestClient. @pytest.fixture(scope="class") -def fastapi_client_class_scope(): - server = setup_server(model_repository=TEST_MODEL_REPOSITORY) - app = setup_fastapi_app( - tokenizer=TEST_TOKENIZER, server=server, backend=TEST_BACKEND - ) +def fastapi_client_class_scope( + model_repository: str, tokenizer_model: str, backend: str +): + server = setup_server(model_repository=model_repository) + app = setup_fastapi_app(tokenizer=tokenizer_model, server=server, backend=backend) with TestClient(app) as test_client: yield test_client server.stop() -@pytest.fixture(scope="module") -def model_repository(): - return TEST_MODEL_REPOSITORY - - -@pytest.fixture(scope="module") -def model(): - return TEST_MODEL - - -@pytest.fixture(scope="module") -def backend(): - return TEST_BACKEND - - -@pytest.fixture(scope="module") -def tokenizer_model(): - return TEST_TOKENIZER - - -@pytest.fixture(scope="module") -def prompt(): - return TEST_PROMPT - - -@pytest.fixture(scope="module") -def messages(): - return TEST_MESSAGES - - # FIXME: In TRTLLM tests, the in-process Triton server for the FastAPI app # does not automatically release GPU memory, even after calling stop(). # The memory is only released when the entire pytest process exits. diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py new file mode 100644 index 0000000000..c8fbfa75ad --- /dev/null +++ b/python/openai/tests/test_embeddings.py @@ -0,0 +1,553 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import base64 +from pathlib import Path + +import numpy as np +import pytest +from this import d + +EMBEDDING_OUTPUT_FLOAT = [ + -0.1914404183626175, + 0.4000193178653717, + 0.058502197265625, + 0.18909454345703125, + -0.4690297544002533, + 0.004936536308377981, + 0.45893096923828125, + -0.31141534447669983, + 0.18299102783203125, + -0.4907582700252533, + 0.6920369267463684, + -0.001537322998046875, + 0.1219015121459961, + -0.11682561784982681, + 0.02811431884765625, + -0.5207672119140625, + 0.4574941098690033, + -0.31097412109375, + 0.13371849060058594, + -0.4693959653377533, + -0.2766602337360382, + 0.029005050659179688, + -0.13730454444885254, + -0.18662643432617188, + 0.0063533782958984375, + 0.16905848681926727, + 0.1612701416015625, + 0.08376502990722656, + -0.09822845458984375, + -0.012738545425236225, + -0.16643650829792023, + -0.01901753805577755, + -0.0503285713493824, + 0.03994830325245857, + -0.3373819887638092, + -0.0188166294246912, + 0.374481201171875, + -0.4371846616268158, + 0.22470474243164062, + -0.011973063461482525, + 0.13784568011760712, + -0.1484222412109375, + 0.19347667694091797, + 0.11848513036966324, + 0.09258091449737549, + -0.2887814939022064, + -0.2301533967256546, + -0.12088584899902344, + 0.1941477507352829, + -0.05228869244456291, + -0.2508443295955658, + 0.15698719024658203, + -0.19024403393268585, + -0.4728952944278717, + 0.336700439453125, + 0.11721674352884293, + 0.24498240649700165, + -0.4826914370059967, + -0.119984470307827, + 0.1008249893784523, + -0.0983428955078125, + -0.0059178671799600124, + 0.2341969758272171, + 0.2725118100643158, + 0.2384185791015625, + -0.30748113989830017, + -0.17387008666992188, + -0.44342041015625, + -0.15537135303020477, + 0.27348455786705017, + 0.1131540909409523, + -0.23855717480182648, + 0.2574056088924408, + -0.12090619653463364, + -0.14412720501422882, + -0.2408244013786316, + 0.0021938085556030273, + -0.39945730566978455, + -0.555908203125, + 0.0760548934340477, + -0.1530914306640625, + -0.40975189208984375, + -0.2091045379638672, + 0.20317332446575165, + -0.20295588672161102, + -0.3643442690372467, + 0.05287488177418709, + -0.24874623119831085, + 0.11500009149312973, + 0.1661122590303421, + 0.26618704199790955, + -0.22980372607707977, + -0.202911376953125, + -0.2738393247127533, + 0.20629756152629852, + -0.24571101367473602, + -0.1486002653837204, + -0.12444128841161728, + 0.27539315819740295, + 0.41679826378822327, + -0.01199467945843935, + 0.1778361052274704, + -0.15123574435710907, + -0.0391184501349926, + -0.035979270935058594, + 0.11838880926370621, + -0.07832065969705582, + 0.15302227437496185, + -0.11540285497903824, + -0.008619308471679688, + 0.011735956184566021, + 0.41825103759765625, + 0.1798756867647171, + 0.0468953438103199, + -0.31410470604896545, + -0.28439536690711975, + 0.028476715087890625, + -0.18972015380859375, + -0.1492512971162796, + 0.23354721069335938, + 0.2631734311580658, + 0.3009694516658783, + -0.31204381585121155, + 0.17155838012695312, + -0.6126009821891785, + -0.16471035778522491, + 0.7154337763786316, + 0.0, + -0.3936564028263092, + -0.15255196392536163, + 0.24118296802043915, + -0.13930638134479523, + 0.6811599731445312, + 0.135009765625, + -0.18750762939453125, + 0.26521047949790955, + -0.1257190704345703, + 0.0532684326171875, + 0.25982680916786194, + -0.3410797119140625, + -0.189666748046875, + 0.016697248443961143, + 0.1474812775850296, + 0.085713230073452, + -0.0862935408949852, + 0.521209716796875, + 0.3840688169002533, + 0.04953320696949959, + -0.0478159599006176, + -0.3888498842716217, + 0.3243462145328522, + 0.03093973733484745, + -0.3594563901424408, + 0.16615693271160126, + -0.07209650427103043, + 0.049218177795410156, + 0.14628247916698456, + -0.10561561584472656, + 0.1696879118680954, + 0.1195220947265625, + 0.0140139264985919, + 0.08987680822610855, + 0.02198282815515995, + -0.06835142523050308, + -0.09100532531738281, + -0.3970082700252533, + -0.20552189648151398, + -0.0871327742934227, + -0.008806228637695312, + 0.10437265783548355, + 0.2754974365234375, + 0.2630208432674408, + -0.67779541015625, + 0.32654380798339844, + -0.4008077085018158, + 0.2785542905330658, + 0.16632080078125, + -0.0709940567612648, + -0.1678619384765625, + -0.11333879083395004, + 0.5577189326286316, + 0.3165779113769531, + -0.2243397980928421, + 0.08053144067525864, + 0.1904652863740921, + 0.22478973865509033, + 0.11852264404296875, + -0.2071024626493454, + 0.2380015105009079, + 0.4622955322265625, + 0.1029459610581398, + -0.30094656348228455, + 0.0351104736328125, + -0.09827486425638199, + 0.0018183389911428094, + 0.07406362146139145, + 0.18090057373046875, + 0.2231648713350296, + -0.1001536026597023, + 0.06609535217285156, + 0.0055376687087118626, + -0.02939859963953495, + -0.17679977416992188, + 0.2300567626953125, + -0.232757568359375, + -0.1863892823457718, + 0.14040501415729523, + -0.21081669628620148, + 0.4772237241268158, + 0.00708770751953125, + 0.25393548607826233, + -0.12926609814167023, + -0.21408335864543915, + 0.43414306640625, + -0.16021983325481415, + -0.6590754389762878, + 0.383026123046875, + 0.4894002377986908, + -0.5350291132926941, + 0.1563262939453125, + 0.4013887941837311, + -0.1429697722196579, + -0.1266673356294632, + 0.0, + -0.12781651318073273, + 0.5082905888557434, + -0.4895477294921875, + 0.05857785418629646, + -0.01038360595703125, + -0.4025942385196686, + -0.6376139521598816, + -0.27256616950035095, + -0.2183430939912796, + 0.13019943237304688, + -0.2378387451171875, + -0.12579791247844696, + 0.23233287036418915, + -0.1948690414428711, + -0.10780048370361328, + 0.4768002927303314, + 0.2761942446231842, + 0.09968694299459457, + -0.07807016372680664, + 0.18632762134075165, + -0.014780680648982525, + 0.18301646411418915, + 0.10943603515625, + 0.45223236083984375, + -0.24634425342082977, + 0.5127970576286316, + 0.15272267162799835, + 0.26901498436927795, + -0.8670451045036316, + -0.20471616089344025, + 0.3934173583984375, + -0.22558848559856415, + 0.14676158130168915, + -0.16282017529010773, + 0.0047810873948037624, + 0.49467912316322327, + -0.1040293350815773, + -0.13565094769001007, + -0.05704273656010628, + 0.2030487060546875, + 0.27226924896240234, + -0.16900062561035156, + 0.06879997253417969, + 0.44347524642944336, + 0.08619359880685806, + -0.1269734650850296, + -0.05267079547047615, + -0.3465728759765625, + 0.1846415251493454, + -0.0655873641371727, + 0.027518590912222862, + -0.06689834594726562, + -0.13316090404987335, + -0.3649355471134186, + -0.0573628731071949, + 0.030780792236328125, + 0.2462870329618454, + -0.0250523891299963, + 0.08964482694864273, + -0.34076571464538574, + -0.3342704772949219, + -0.000331878662109375, + 0.25020280480384827, + 0.34731578826904297, + 0.4081510007381439, + 0.0661773681640625, + 0.14612038433551788, + -0.37111154198646545, + -0.17901070415973663, + 0.0565798282623291, + -0.1689503937959671, + 0.311676025390625, + 0.06296539306640625, + 0.11648496240377426, + -0.16365115344524384, + -0.011795361526310444, + -0.4601001739501953, + 0.13840866088867188, + 0.1115519180893898, + -0.3645426332950592, + -0.182403564453125, + -0.20782725512981415, + -0.004481792449951172, + 0.0870104655623436, + -0.11704126745462418, + 0.34148290753364563, + 0.17841561138629913, + -0.2754109799861908, + -0.0867462158203125, + 0.09910837560892105, + -0.14540545642375946, + -0.10996246337890625, + -0.10946687310934067, + 0.023001352325081825, + 0.11987527459859848, + -5.960464477539063e-8, + 0.3316993713378906, + -0.025622526183724403, + -0.28015899658203125, + 0.34741735458374023, + 0.04091135784983635, + -0.34874120354652405, + 0.22758229076862335, + -0.042999267578125, + 0.0382130928337574, + 0.5654922127723694, + -0.9378255009651184, + 0.17114512622356415, + 0.13035202026367188, + 0.4369252622127533, + 0.0897369384765625, + 0.19928233325481415, + 0.33091607689857483, + -0.10624822229146957, + -0.2845611572265625, + 0.2822163999080658, + 0.1722426414489746, + 0.2111460417509079, + -0.1069692000746727, + -0.3496347963809967, + 0.15000660717487335, + 0.014147520065307617, + -0.36633554100990295, + 0.23989041149616241, + -0.06397350877523422, + 0.2501627504825592, + 0.04016287997364998, + -0.3789469301700592, + -0.4247843325138092, + 0.1515035629272461, + 0.36554718017578125, + 0.057392120361328125, + -0.3492482602596283, + -0.45532989501953125, + 0.4090474545955658, + -0.3914286196231842, + -0.4888407289981842, + 0.4746551513671875, + -0.6188761591911316, + -0.018857955932617188, + 0.02373504638671875, + 0.22691090404987335, + -0.07608286291360855, + 0.5331514477729797, + -0.27182260155677795, + 0.2309315949678421, + -0.1824493408203125, + 0.12648265063762665, + 0.2586142122745514, + -0.07648912817239761, + 0.2318166047334671, + -0.5225245356559753, + 0.133880615234375, + -0.010974247939884663, + 0.09001413732767105, + 0.2562611997127533, + 0.19260406494140625, + 0.4470011293888092, + -0.1636505126953125, + -0.3675130307674408, +] + + +@pytest.mark.fastapi +class TestEmbeddings: + @pytest.fixture(scope="class", autouse=True) + def check_backend(self, backend: str): + if backend != "vllm": + pytest.skip("These tests only run with vLLM backend") + + @pytest.fixture(scope="class") + def client(self, fastapi_client_class_scope): + yield fastapi_client_class_scope + + @pytest.fixture(scope="class") + def model(self): + # Override with embeddings-specific model + return "all-MiniLM-L6-v2" + + @pytest.fixture(scope="class") + def model_repository(self): + # Override with embeddings-specific repository + return str(Path(__file__).parent / "vllm_embedding_models") + + @pytest.fixture(scope="class") + def input(self): + return "The food was delicious and the waiter..." + + def _check_embedding_response( + self, response, model, dims=len(EMBEDDING_OUTPUT_FLOAT), encoding_format="float" + ): + assert response.status_code == 200, response.json() + embedding = response.json()["data"][0]["embedding"] + assert embedding is not None + if encoding_format == "base64": + embedding = np.frombuffer(base64.b64decode(embedding), dtype=np.float32) + result = np.allclose( + EMBEDDING_OUTPUT_FLOAT[:dims], embedding, rtol=1e-5, atol=0.0 + ) + assert result, "Embedding does not match expected output" + + assert response.json()["data"][0]["object"] == "embedding" + assert response.json()["data"][0]["index"] == 0 + + assert response.json()["model"] == model + + usage = response.json().get("usage") + assert usage is not None + assert usage["prompt_tokens"] == 12 + assert usage["total_tokens"] == 12 + + @pytest.mark.parametrize( + "input", + [ + "The food was delicious and the waiter...", + [101, 1996, 2833, 2001, 12090, 1998, 1996, 15610, 1012, 1012, 1012, 102], + ], + ) + def test_embeddings_defaults(self, client, model: str, input: str): + response = client.post( + "/v1/embeddings", + json={"model": model, "input": input}, + ) + + self._check_embedding_response(response, model) + + # TODO: Python model cannot unload gracefully if raise error. + # def test_chat_completions_defaults( + # self, client, model: str, messages: List[dict], backend: str + # ): + # response = client.post( + # "/v1/chat/completions", + # json={"model": model, "messages": messages}, + # ) + + # assert response.status_code == 400 + # assert "does not support" in response.json()["detail"] + + @pytest.mark.parametrize( + "param_key, param_value", + [ + ("dimensions", [10]), + ("encoding_format", "invalid"), + ("encoding_format", 0), + ], + ) + def test_embeddings_invalid_parameters( + self, client, param_key, param_value, model: str, input: str + ): + response = client.post( + "/v1/embeddings", + json={ + "model": model, + "input": input, + param_key: param_value, + }, + ) + + # Assert schema validation error + assert response.status_code == 422, response.json() + + @pytest.mark.parametrize("dimensions", [0, 10, 100, -1]) + @pytest.mark.parametrize("encoding_format", ["float", "base64"]) + def test_embeddings_parameters( + self, client, dimensions, encoding_format, model: str, input: str + ): + response = client.post( + "/v1/embeddings", + json={ + "model": model, + "input": input, + "dimensions": dimensions, + "encoding_format": encoding_format, + }, + ) + + self._check_embedding_response( + response, model, dims=dimensions, encoding_format=encoding_format + ) + + @pytest.mark.parametrize( + "input", + [ + "", + [], + ], + ) + def test_embeddings_no_input(self, client, model: str, input): + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == 422 + assert ( + response.json()["detail"][0]["msg"] + == "Value should have at least 1 item after validation, not 0" + ) From 34808deeb02d3ec30af35bfb6fd21812928b998f Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Wed, 29 Oct 2025 21:53:00 -0700 Subject: [PATCH 10/21] security: Update package version due to CVE-2025-62727 (#8482) --- python/openai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index 608f3b5ac3..e523449265 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -33,4 +33,4 @@ openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs # Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw -starlette>=0.40.0 +starlette>=0.49.1 From 8fccf3ab4dc32abc0c916ae2b1d9cde62a821c1f Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 30 Oct 2025 09:08:30 -0700 Subject: [PATCH 11/21] model --- .../all-MiniLM-L6-v2/1/model.json | 1 + .../all-MiniLM-L6-v2/config.pbtxt | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json create mode 100644 python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json new file mode 100644 index 0000000000..2ad058c2e5 --- /dev/null +++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json @@ -0,0 +1 @@ +{"model": "sentence-transformers/all-MiniLM-L6-v2", "gpu_memory_utilization": 0.5} diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt new file mode 100644 index 0000000000..39b3c48edb --- /dev/null +++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt @@ -0,0 +1,28 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +backend: "vllm" +instance_group [{kind: KIND_MODEL}] From 76eeeec3a5523b0e12b0d3248d06fe62b3f1787c Mon Sep 17 00:00:00 2001 From: Misha Chornyi <99709299+mc-nv@users.noreply.github.com> Date: Thu, 30 Oct 2025 09:24:29 -0700 Subject: [PATCH 12/21] Revert "security: Update package version due to CVE-2025-62727" (#8484) --- python/openai/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index e523449265..608f3b5ac3 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -33,4 +33,4 @@ openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs # Minimum starlette version needed to address CVE: # https://github.com/advisories/GHSA-f96h-pmfr-66vw -starlette>=0.49.1 +starlette>=0.40.0 From 876582be6bdf0f9ce53b81881695a393a14f00fb Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 30 Oct 2025 14:59:41 -0700 Subject: [PATCH 13/21] Minor fix --- python/openai/openai_frontend/engine/triton_engine.py | 2 +- python/openai/tests/conftest.py | 2 +- python/openai/tests/test_embeddings.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 3f8ca8cff6..d47205ffa3 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -410,7 +410,7 @@ def _get_embedding( ) -> Union[list[float], str]: if encoding_format == "float": return embedding - elif encoding_format == "base64": + else: embedding_bytes = np.array(embedding, dtype="float32").tobytes() return base64.b64encode(embedding_bytes).decode("utf-8") diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index d05ce697c0..5c269a78c1 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -87,7 +87,7 @@ def backend(tool_call_parser): @pytest.fixture(scope="session") -def model(backend, tool_call_parser): +def model(tool_call_parser): env_model = os.environ.get("TEST_MODEL") if not env_model: diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py index c8fbfa75ad..d1c70126d1 100644 --- a/python/openai/tests/test_embeddings.py +++ b/python/openai/tests/test_embeddings.py @@ -29,7 +29,6 @@ import numpy as np import pytest -from this import d EMBEDDING_OUTPUT_FLOAT = [ -0.1914404183626175, From 3c3d6ccc1ac9cb655e91c5b946d2d6d67d839ba5 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Fri, 31 Oct 2025 14:40:34 -0700 Subject: [PATCH 14/21] Add example document --- python/openai/README.md | 92 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/python/openai/README.md b/python/openai/README.md index 29ee39a194..bcabf5e969 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -301,6 +301,98 @@ See the [vLLM documentation](https://github.com/triton-inference-server/vllm_backend/blob/main/docs/llama_multi_lora_tutorial.md) on how to serve a model with LoRA adapters. +### Embedding Models +Currently, OpenAI-Compatible Frontend supports loading embedding models and embeddings endpoints via vLLM backend. Check [vLLM supported models](https://docs.vllm.ai/en/latest/models/supported_models.html#embedding) for all supported embedding models from vLLM. + +1. Launch the container and install dependencies: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 +``` + +2. Launch the OpenAI-compatible Triton Inference Server: +```bash +cd /opt/tritonserver/python/openai + +# NOTE: Embeddings endpoint does not require "--tokenizer" +python3 openai_frontend/main.py --model-repository tests/vllm_embedding_models +``` + +
+Example output + +``` +... ++------------------+---------+--------+ +| Model | Version | Status | ++------------------+---------+--------+ +| all-MiniLM-L6-v2 | 1 | READY | <- Correct Model Loaded in Triton ++------------------+---------+--------+ +... +Found model: name='all-MiniLM-L6-v2', backend='vllm' +[WARNING] Adding CORS for the following origins: ['http://localhost'] +INFO: Started server process [133] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully +``` + +
+ +3. Send a `/v1/embeddings` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="all-MiniLM-L6-v2" +curl -s http://localhost:9000/v1/embeddings \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "'${MODEL}'", + "input": "The food was delicious and the waiter...", + "dimensions": 10, + "encoding_format": "float" + }' | jq +``` + +
+Example output + +```json +{ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + -0.1914404183626175, + 0.4000193178653717, + 0.058502197265625, + 0.18909454345703125, + -0.4690297544002533, + 0.004936536308377981, + 0.45893096923828125, + -0.31141534447669983, + 0.18299102783203125, + -0.4907582700252533 + ], + "index": 0 + } + ], + "model": "all-MiniLM-L6-v2", + "usage": { + "prompt_tokens": 12, + "total_tokens": 12 + } +} +``` + +
+ ## TensorRT-LLM 0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: From 05e0f98f0bdd0ccd2003b0880460b0c48dfa8b9d Mon Sep 17 00:00:00 2001 From: Yingge He Date: Mon, 3 Nov 2025 10:14:34 -0800 Subject: [PATCH 15/21] Fix conflict --- TRITON_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TRITON_VERSION b/TRITON_VERSION index 995366c63c..867f66f09b 100644 --- a/TRITON_VERSION +++ b/TRITON_VERSION @@ -1 +1 @@ -2.62.0 +2.63.0dev From 458b8454a713b985ccfb5d951f6c0e6869bc0912 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 4 Nov 2025 09:06:38 -0800 Subject: [PATCH 16/21] Update naming --- python/openai/openai_frontend/engine/engine.py | 2 +- python/openai/openai_frontend/engine/triton_engine.py | 4 +++- python/openai/openai_frontend/engine/utils/triton.py | 4 ++-- .../openai_frontend/frontend/fastapi/routers/embeddings.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/python/openai/openai_frontend/engine/engine.py b/python/openai/openai_frontend/engine/engine.py index 527ef9542d..2dfeafb1db 100644 --- a/python/openai/openai_frontend/engine/engine.py +++ b/python/openai/openai_frontend/engine/engine.py @@ -95,7 +95,7 @@ def completion( """ pass - def embed(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: + def embedding(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: """ Returns a CreateEmbeddingResponse. """ diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index d47205ffa3..492804ede1 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -367,7 +367,9 @@ async def completion( usage=usage, ) - async def embed(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: + async def embedding( + self, request: CreateEmbeddingRequest + ) -> CreateEmbeddingResponse: # Validate request and convert to Triton format model_name, _ = self._get_model_and_lora_name(request.model) metadata = self.model_metadata.get(model_name) diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 35aa75fea0..542d4d964a 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -47,8 +47,8 @@ class RequestKind(Enum): - GENERATE = "generate" - EMBED = "embed" + GENERATE = 1 + EMBED = 2 def _create_vllm_generate_request( diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py index 9da54b56cb..84f2604d21 100644 --- a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py +++ b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py @@ -44,7 +44,7 @@ async def create_embedding( raise HTTPException(status_code=500, detail="No attached inference engine") try: - response = await raw_request.app.engine.embed(request) + response = await raw_request.app.engine.embedding(request) return response except Exception as e: raise HTTPException(status_code=400, detail=f"{e}") From 87ca9bc52371455213e37b597013b355282eeb57 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 4 Nov 2025 21:54:52 -0800 Subject: [PATCH 17/21] More tests --- python/openai/tests/test_embeddings.py | 54 +++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py index d1c70126d1..77bd677749 100644 --- a/python/openai/tests/test_embeddings.py +++ b/python/openai/tests/test_embeddings.py @@ -481,7 +481,7 @@ def test_embeddings_defaults(self, client, model: str, input: str): self._check_embedding_response(response, model) - # TODO: Python model cannot unload gracefully if raise error. + # FIXME: Python model cannot unload gracefully if raise error. # def test_chat_completions_defaults( # self, client, model: str, messages: List[dict], backend: str # ): @@ -535,6 +535,41 @@ def test_embeddings_parameters( response, model, dims=dimensions, encoding_format=encoding_format ) + def test_embeddings_empty_request(self, client): + response = client.post("/v1/embeddings", json={}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Field required" + + def test_embeddings_no_model(self, client, input: str): + response = client.post("/v1/embeddings", json={"input": input}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Field required" + + @pytest.mark.parametrize( + "model, error_code", + [ + ("", 400), + (123, 422), + ("Invalid", 400), + (None, 422), + ], + ) + def test_embeddings_invalid_model(self, client, model: str, input, error_code: int): + print("Model:", model) + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == error_code + if error_code == 400: + assert response.json()["detail"] == f"Unknown model: {model}" + else: + assert ( + response.json()["detail"][0]["msg"] == "Input should be a valid string" + ) + + def test_embeddings_no_input(self, client, model: str): + response = client.post("/v1/embeddings", json={"model": model}) + assert response.status_code == 422 + @pytest.mark.parametrize( "input", [ @@ -542,7 +577,7 @@ def test_embeddings_parameters( [], ], ) - def test_embeddings_no_input(self, client, model: str, input): + def test_embeddings_empty_input(self, client, model: str, input): # Message validation requires min_length of 1 response = client.post("/v1/embeddings", json={"model": model, "input": input}) assert response.status_code == 422 @@ -550,3 +585,18 @@ def test_embeddings_no_input(self, client, model: str, input): response.json()["detail"][0]["msg"] == "Value should have at least 1 item after validation, not 0" ) + + @pytest.mark.parametrize( + "input", + [ + 123, + 1.5, + 0, + None, + ], + ) + def test_embeddings_invalid_input(self, client, model: str, input): + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Input should be a valid string" From 61076d9bbf7c6e899731c96598843c8aec014819 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 4 Nov 2025 22:10:23 -0800 Subject: [PATCH 18/21] Fix trt tests --- python/openai/tests/test_embeddings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py index 77bd677749..4ae5e4516e 100644 --- a/python/openai/tests/test_embeddings.py +++ b/python/openai/tests/test_embeddings.py @@ -421,9 +421,11 @@ @pytest.mark.fastapi class TestEmbeddings: @pytest.fixture(scope="class", autouse=True) - def check_backend(self, backend: str): + def check_backend(backend: str): if backend != "vllm": - pytest.skip("These tests only run with vLLM backend") + pytest.skip( + "TRT-LLM backend and Python backend do not support embedding requests" + ) @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): From 976ca524cbb5db7249bb8afc8f9fa1c13651c5a2 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 4 Nov 2025 23:02:02 -0800 Subject: [PATCH 19/21] Update tests --- python/openai/tests/test_chat_completions.py | 2 -- python/openai/tests/test_completions.py | 1 - python/openai/tests/test_embeddings.py | 13 +++++-------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 5402be451d..565146f7c7 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -35,7 +35,6 @@ from tests.utils import setup_fastapi_app, setup_server -@pytest.mark.fastapi class TestChatCompletions: @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): @@ -564,7 +563,6 @@ def test_usage_response( # For tests that won't use the same pytest fixture for server startup across # the whole class test suite. -@pytest.mark.fastapi class TestChatCompletionsTokenizers: # Re-use a single Triton server for different frontend configurations @pytest.fixture(scope="class") diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 9ec3ffe7f7..31eb2f8ae6 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -29,7 +29,6 @@ import pytest -@pytest.mark.fastapi class TestCompletions: @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py index 4ae5e4516e..a64e21d496 100644 --- a/python/openai/tests/test_embeddings.py +++ b/python/openai/tests/test_embeddings.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base64 +import os from pathlib import Path import numpy as np @@ -418,15 +419,11 @@ ] -@pytest.mark.fastapi +@pytest.mark.skipif( + os.environ.get("IMAGE_KIND") == "TRTLLM", + reason="TRT-LLM backend does not support embedding requests", +) class TestEmbeddings: - @pytest.fixture(scope="class", autouse=True) - def check_backend(backend: str): - if backend != "vllm": - pytest.skip( - "TRT-LLM backend and Python backend do not support embedding requests" - ) - @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): yield fastapi_client_class_scope From b48425ef5bea35753810366d8bf35781099fa23f Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 4 Nov 2025 23:19:32 -0800 Subject: [PATCH 20/21] Update naming --- .../openai_frontend/engine/triton_engine.py | 16 +++++++++------- .../openai_frontend/engine/utils/triton.py | 10 +++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 492804ede1..f0d184cca2 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -244,7 +244,7 @@ async def chat( ) usage = _get_usage_from_response( - response, metadata.backend, RequestKind.GENERATE + response, metadata.backend, RequestKind.GENERATION ) return CreateChatCompletionResponse( @@ -348,7 +348,7 @@ async def completion( text = _get_output(response) usage = _get_usage_from_response( - response, metadata.backend, RequestKind.GENERATE + response, metadata.backend, RequestKind.GENERATION ) choice = Choice( @@ -392,7 +392,9 @@ async def embedding( embedding_json = _get_output(response) embedding_list = json.loads(embedding_json) - usage = _get_usage_from_response(response, metadata.backend, RequestKind.EMBED) + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.EMBEDDING + ) embedding = self._get_embedding(embedding_list, request.encoding_format) embedding_obj = EmbeddingObject( @@ -433,14 +435,14 @@ def _determine_request_converter(self, backend: str, request_type: RequestKind): # Request conversion from OpenAI format to backend-specific format if backend == "vllm": - if request_type == RequestKind.GENERATE: + if request_type == RequestKind.GENERATION: return _create_vllm_generate_request else: return _create_vllm_embedding_request # Use TRT-LLM format as default for everything else. This could be # an ensemble, a python or BLS model, a TRT-LLM backend model, etc. - if request_type == RequestKind.GENERATE: + if request_type == RequestKind.GENERATION: return _create_trtllm_generate_request else: return _create_trtllm_embedding_request @@ -489,10 +491,10 @@ def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]: lora_names=lora_names, create_time=self.create_time, inference_request_converter=self._determine_request_converter( - backend, RequestKind.GENERATE + backend, RequestKind.GENERATION ), embedding_request_converter=self._determine_request_converter( - backend, RequestKind.EMBED + backend, RequestKind.EMBEDDING ), ) model_metadata[name] = metadata diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 542d4d964a..6258c4b9df 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -47,8 +47,8 @@ class RequestKind(Enum): - GENERATE = 1 - EMBED = 2 + GENERATION = 1 + EMBEDDING = 2 def _create_vllm_generate_request( @@ -273,7 +273,7 @@ class _StreamingUsageAccumulator: def update(self, response: tritonserver.InferenceResponse): """Extracts usage from a response and updates the token counts.""" - usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATE) + usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATION) if usage: # The prompt_tokens is received with every chunk but should only be set once. if not self._prompt_tokens_set: @@ -332,14 +332,14 @@ def _get_usage_from_response( completion_tokens = completion_tokens_ptr[0] if prompt_tokens is not None: - if request_type == RequestKind.GENERATE and completion_tokens is not None: + if request_type == RequestKind.GENERATION and completion_tokens is not None: total_tokens = prompt_tokens + completion_tokens return CompletionUsage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, ) - elif request_type == RequestKind.EMBED: + elif request_type == RequestKind.EMBEDDING: return EmbeddingUsage( prompt_tokens=prompt_tokens, total_tokens=prompt_tokens, From c022c3e31880f1cf8c28e34e9a452f7c6b2f6f5f Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 5 Nov 2025 14:43:30 -0800 Subject: [PATCH 21/21] Fix issue on A100 runner --- python/openai/tests/test_embeddings.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py index a64e21d496..b2a21b78e9 100644 --- a/python/openai/tests/test_embeddings.py +++ b/python/openai/tests/test_embeddings.py @@ -31,6 +31,7 @@ import numpy as np import pytest +# Results on A6000 GPU. The results vary slightly across GPU models. EMBEDDING_OUTPUT_FLOAT = [ -0.1914404183626175, 0.4000193178653717, @@ -433,6 +434,10 @@ def model(self): # Override with embeddings-specific model return "all-MiniLM-L6-v2" + @pytest.fixture(scope="class") + def tokenizer_model(self): + return None + @pytest.fixture(scope="class") def model_repository(self): # Override with embeddings-specific repository @@ -450,14 +455,17 @@ def _check_embedding_response( assert embedding is not None if encoding_format == "base64": embedding = np.frombuffer(base64.b64decode(embedding), dtype=np.float32) + + # The results vary slightly across GPU models result = np.allclose( - EMBEDDING_OUTPUT_FLOAT[:dims], embedding, rtol=1e-5, atol=0.0 + EMBEDDING_OUTPUT_FLOAT[:dims], embedding, rtol=0, atol=1e-3 ) - assert result, "Embedding does not match expected output" + assert ( + result + ), f"Embeddings do not match expected output\nExpect {EMBEDDING_OUTPUT_FLOAT[:dims]},\ngot{embedding}" assert response.json()["data"][0]["object"] == "embedding" assert response.json()["data"][0]["index"] == 0 - assert response.json()["model"] == model usage = response.json().get("usage")