diff --git a/python/openai/README.md b/python/openai/README.md
index 572373390f..bcabf5e969 100644
--- a/python/openai/README.md
+++ b/python/openai/README.md
@@ -301,6 +301,98 @@ See the
[vLLM documentation](https://github.com/triton-inference-server/vllm_backend/blob/main/docs/llama_multi_lora_tutorial.md)
on how to serve a model with LoRA adapters.
+### Embedding Models
+Currently, OpenAI-Compatible Frontend supports loading embedding models and embeddings endpoints via vLLM backend. Check [vLLM supported models](https://docs.vllm.ai/en/latest/models/supported_models.html#embedding) for all supported embedding models from vLLM.
+
+1. Launch the container and install dependencies:
+ - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc.
+ - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to
+ access gated models, make sure this is set in your local environment if needed.
+
+```bash
+docker run -it --net=host --gpus all --rm \
+ -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \
+ -e HF_TOKEN \
+ nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3
+```
+
+2. Launch the OpenAI-compatible Triton Inference Server:
+```bash
+cd /opt/tritonserver/python/openai
+
+# NOTE: Embeddings endpoint does not require "--tokenizer"
+python3 openai_frontend/main.py --model-repository tests/vllm_embedding_models
+```
+
+
+Example output
+
+```
+...
++------------------+---------+--------+
+| Model | Version | Status |
++------------------+---------+--------+
+| all-MiniLM-L6-v2 | 1 | READY | <- Correct Model Loaded in Triton
++------------------+---------+--------+
+...
+Found model: name='all-MiniLM-L6-v2', backend='vllm'
+[WARNING] Adding CORS for the following origins: ['http://localhost']
+INFO: Started server process [133]
+INFO: Waiting for application startup.
+INFO: Application startup complete.
+INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully
+```
+
+
+
+3. Send a `/v1/embeddings` request:
+ - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses.
+```bash
+MODEL="all-MiniLM-L6-v2"
+curl -s http://localhost:9000/v1/embeddings \
+ -H 'Content-Type: application/json' \
+ -d '{
+ "model": "'${MODEL}'",
+ "input": "The food was delicious and the waiter...",
+ "dimensions": 10,
+ "encoding_format": "float"
+ }' | jq
+```
+
+
+Example output
+
+```json
+{
+ "object": "list",
+ "data": [
+ {
+ "object": "embedding",
+ "embedding": [
+ -0.1914404183626175,
+ 0.4000193178653717,
+ 0.058502197265625,
+ 0.18909454345703125,
+ -0.4690297544002533,
+ 0.004936536308377981,
+ 0.45893096923828125,
+ -0.31141534447669983,
+ 0.18299102783203125,
+ -0.4907582700252533
+ ],
+ "index": 0
+ }
+ ],
+ "model": "all-MiniLM-L6-v2",
+ "usage": {
+ "prompt_tokens": 12,
+ "total_tokens": 12
+ }
+}
+```
+
+
+
## TensorRT-LLM
0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options:
@@ -655,6 +747,8 @@ Use the `--openai-restricted-api` command-line argument to configure endpoint re
- **inference**: Chat completions and text completions endpoints
- `POST /v1/chat/completions`
- `POST /v1/completions`
+ - **embedding**: Embedding endpoint
+ - `POST /v1/embeddings`
- **model-repository**: Model listing and information endpoints
- `GET /v1/models`
- `GET /v1/models/{model_name}`
diff --git a/python/openai/openai_frontend/engine/engine.py b/python/openai/openai_frontend/engine/engine.py
index 9c90dec25e..2dfeafb1db 100644
--- a/python/openai/openai_frontend/engine/engine.py
+++ b/python/openai/openai_frontend/engine/engine.py
@@ -1,4 +1,4 @@
-# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
@@ -34,6 +34,8 @@
CreateChatCompletionResponse,
CreateCompletionRequest,
CreateCompletionResponse,
+ CreateEmbeddingRequest,
+ CreateEmbeddingResponse,
Model,
)
@@ -92,3 +94,9 @@ def completion(
If request.stream is False, this returns a CreateCompletionResponse.
"""
pass
+
+ def embedding(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse:
+ """
+ Returns a CreateEmbeddingResponse.
+ """
+ pass
diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py
index 499cc623e7..f0d184cca2 100644
--- a/python/openai/openai_frontend/engine/triton_engine.py
+++ b/python/openai/openai_frontend/engine/triton_engine.py
@@ -27,6 +27,7 @@
from __future__ import annotations
+import base64
import json
import time
import uuid
@@ -38,18 +39,24 @@
Callable,
Dict,
List,
+ Literal,
Optional,
Tuple,
+ Union,
)
+import numpy as np
import tritonserver
from engine.engine import LLMEngine
from engine.utils.chat import load_chat_template, parse_chat_messages
from engine.utils.tokenizer import get_tokenizer
from engine.utils.tool_call_parsers import ToolCallParser, ToolParserManager
from engine.utils.triton import (
- _create_trtllm_inference_request,
- _create_vllm_inference_request,
+ RequestKind,
+ _create_trtllm_embedding_request,
+ _create_trtllm_generate_request,
+ _create_vllm_embedding_request,
+ _create_vllm_generate_request,
_get_output,
_get_usage_from_response,
_get_vllm_lora_names,
@@ -73,6 +80,9 @@
CreateChatCompletionStreamResponse,
CreateCompletionRequest,
CreateCompletionResponse,
+ CreateEmbeddingRequest,
+ CreateEmbeddingResponse,
+ EmbeddingObject,
FinishReason,
Function1,
Function2,
@@ -97,7 +107,8 @@ class TritonModelMetadata:
# Time that model was loaded by Triton
create_time: int
# Conversion format between OpenAI and Triton requests
- request_converter: Callable
+ inference_request_converter: Callable
+ embedding_request_converter: Callable
class TritonLLMEngine(LLMEngine):
@@ -189,7 +200,7 @@ async def chat(
# Convert to Triton request format and perform inference
responses = metadata.model.async_infer(
- metadata.request_converter(
+ metadata.inference_request_converter(
metadata.model, prompt, request, lora_name, self.default_max_tokens
)
)
@@ -232,7 +243,9 @@ async def chat(
backend=metadata.backend,
)
- usage = _get_usage_from_response(response, metadata.backend)
+ usage = _get_usage_from_response(
+ response, metadata.backend, RequestKind.GENERATION
+ )
return CreateChatCompletionResponse(
id=request_id,
@@ -311,7 +324,7 @@ async def completion(
# Convert to Triton request format and perform inference
responses = metadata.model.async_infer(
- metadata.request_converter(
+ metadata.inference_request_converter(
metadata.model,
request.prompt,
request,
@@ -334,7 +347,9 @@ async def completion(
response = responses[0]
text = _get_output(response)
- usage = _get_usage_from_response(response, metadata.backend)
+ usage = _get_usage_from_response(
+ response, metadata.backend, RequestKind.GENERATION
+ )
choice = Choice(
finish_reason=FinishReason.stop,
@@ -352,6 +367,57 @@ async def completion(
usage=usage,
)
+ async def embedding(
+ self, request: CreateEmbeddingRequest
+ ) -> CreateEmbeddingResponse:
+ # Validate request and convert to Triton format
+ model_name, _ = self._get_model_and_lora_name(request.model)
+ metadata = self.model_metadata.get(model_name)
+ self._validate_embedding_request(request, metadata)
+
+ # Convert to Triton request format and perform inference
+ responses = metadata.model.async_infer(
+ metadata.embedding_request_converter(
+ metadata.model,
+ request,
+ )
+ )
+
+ # Response validation with decoupled models in mind
+ responses = [response async for response in responses]
+ _validate_triton_responses_non_streaming(responses)
+ response = responses[0]
+
+ # Extract embedding from response (currently stored as JSON string in text_output)
+ embedding_json = _get_output(response)
+ embedding_list = json.loads(embedding_json)
+
+ usage = _get_usage_from_response(
+ response, metadata.backend, RequestKind.EMBEDDING
+ )
+
+ embedding = self._get_embedding(embedding_list, request.encoding_format)
+ embedding_obj = EmbeddingObject(
+ embedding=embedding, index=0, object="embedding"
+ )
+
+ return CreateEmbeddingResponse(
+ object="list",
+ data=[embedding_obj],
+ model=request.model,
+ usage=usage,
+ )
+
+ @staticmethod
+ def _get_embedding(
+ embedding: List[float], encoding_format: Literal["float", "base64"]
+ ) -> Union[list[float], str]:
+ if encoding_format == "float":
+ return embedding
+ else:
+ embedding_bytes = np.array(embedding, dtype="float32").tobytes()
+ return base64.b64encode(embedding_bytes).decode("utf-8")
+
# TODO: This behavior should be tested further
def _get_first_response_role(
self, conversation: List[Dict], add_generation_prompt: bool, default_role: str
@@ -362,18 +428,24 @@ def _get_first_response_role(
return conversation[-1]["role"]
# TODO: Expose explicit flag to catch edge cases
- def _determine_request_converter(self, backend: str):
+ def _determine_request_converter(self, backend: str, request_type: RequestKind):
# Allow manual override of backend request format if provided by user
if self.backend:
backend = self.backend
# Request conversion from OpenAI format to backend-specific format
if backend == "vllm":
- return _create_vllm_inference_request
+ if request_type == RequestKind.GENERATION:
+ return _create_vllm_generate_request
+ else:
+ return _create_vllm_embedding_request
# Use TRT-LLM format as default for everything else. This could be
# an ensemble, a python or BLS model, a TRT-LLM backend model, etc.
- return _create_trtllm_inference_request
+ if request_type == RequestKind.GENERATION:
+ return _create_trtllm_generate_request
+ else:
+ return _create_trtllm_embedding_request
def _get_model_and_lora_name(self, request_model_name: str):
if self.lora_separator is None or len(self.lora_separator) == 0:
@@ -418,7 +490,12 @@ def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]:
tokenizer=self.tokenizer,
lora_names=lora_names,
create_time=self.create_time,
- request_converter=self._determine_request_converter(backend),
+ inference_request_converter=self._determine_request_converter(
+ backend, RequestKind.GENERATION
+ ),
+ embedding_request_converter=self._determine_request_converter(
+ backend, RequestKind.EMBEDDING
+ ),
)
model_metadata[name] = metadata
@@ -671,8 +748,15 @@ def _validate_chat_request(
if not metadata.backend:
raise Exception("Unknown backend")
- if not metadata.request_converter:
- raise Exception(f"Unknown request format for model: {request.model}")
+ if not metadata.inference_request_converter:
+ raise Exception(
+ f"Unknown inference request format for model: {request.model}"
+ )
+
+ if not metadata.embedding_request_converter:
+ raise Exception(
+ f"Unknown embedding request format for model: {request.model}"
+ )
if (
metadata.lora_names is not None
@@ -807,8 +891,15 @@ def _validate_completion_request(
if not metadata.backend:
raise Exception("Unknown backend")
- if not metadata.request_converter:
- raise Exception(f"Unknown request format for model: {request.model}")
+ if not metadata.inference_request_converter:
+ raise Exception(
+ f"Unknown inference request format for model: {request.model}"
+ )
+
+ if not metadata.embedding_request_converter:
+ raise Exception(
+ f"Unknown embedding request format for model: {request.model}"
+ )
if (
metadata.lora_names is not None
@@ -853,6 +944,32 @@ def _validate_completion_request(
"`stream_options.include_usage` is currently only supported for the vLLM backend"
)
+ def _validate_embedding_request(
+ self,
+ request: CreateEmbeddingRequest,
+ metadata: TritonModelMetadata,
+ ):
+ """
+ Validates an embedding request to align with currently supported features.
+ """
+
+ # Reject missing internal information needed to do inference
+ if not metadata:
+ raise Exception(f"Unknown model: {request.model}")
+
+ if not metadata.backend:
+ raise Exception("Unknown backend")
+
+ if not metadata.inference_request_converter:
+ raise Exception(
+ f"Unknown inference request format for model: {request.model}"
+ )
+
+ if not metadata.embedding_request_converter:
+ raise Exception(
+ f"Unknown embedding request format for model: {request.model}"
+ )
+
def _should_stream_with_auto_tool_parsing(
self, request: CreateChatCompletionRequest
):
diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py
index 636e58435d..6258c4b9df 100644
--- a/python/openai/openai_frontend/engine/utils/triton.py
+++ b/python/openai/openai_frontend/engine/utils/triton.py
@@ -28,6 +28,7 @@
import os
import re
from dataclasses import asdict, dataclass, field
+from enum import Enum
from pathlib import Path
from typing import Iterable, List, Optional, Union
@@ -40,10 +41,17 @@
CompletionUsage,
CreateChatCompletionRequest,
CreateCompletionRequest,
+ CreateEmbeddingRequest,
+ EmbeddingUsage,
)
-def _create_vllm_inference_request(
+class RequestKind(Enum):
+ GENERATION = 1
+ EMBEDDING = 2
+
+
+def _create_vllm_generate_request(
model,
prompt,
request: CreateChatCompletionRequest | CreateCompletionRequest,
@@ -128,7 +136,7 @@ def _create_vllm_inference_request(
return model.create_request(inputs=inputs)
-def _create_trtllm_inference_request(
+def _create_trtllm_generate_request(
model,
prompt,
request: CreateChatCompletionRequest | CreateCompletionRequest,
@@ -183,6 +191,35 @@ def _create_trtllm_inference_request(
return model.create_request(inputs=inputs)
+def _create_vllm_embedding_request(
+ model,
+ request: CreateEmbeddingRequest,
+):
+ inputs = {}
+ embedding_request = {}
+ embedding_request["input"] = request.input
+
+ pooling_params = {}
+ dims = request.dimensions
+ if dims is not None:
+ pooling_params["dimensions"] = [dims]
+ embedding_request["pooling_params"] = pooling_params
+
+ inputs["embedding_request"] = [json.dumps(embedding_request)]
+ inputs["return_num_input_tokens"] = np.bool_([True])
+ inputs["return_num_output_tokens"] = np.bool_([True])
+ return model.create_request(inputs=inputs)
+
+
+def _create_trtllm_embedding_request(
+ model,
+ request: CreateEmbeddingRequest,
+):
+ raise Exception(
+ "TRT-LLM backend and Python backend do not support embedding requests"
+ )
+
+
def _construct_string_from_pointer(pointer: int, size: int) -> str:
"""Constructs a Python string from a C pointer and size."""
@@ -236,7 +273,7 @@ class _StreamingUsageAccumulator:
def update(self, response: tritonserver.InferenceResponse):
"""Extracts usage from a response and updates the token counts."""
- usage = _get_usage_from_response(response, self.backend)
+ usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATION)
if usage:
# The prompt_tokens is received with every chunk but should only be set once.
if not self._prompt_tokens_set:
@@ -262,7 +299,8 @@ def get_final_usage(self) -> Optional[CompletionUsage]:
def _get_usage_from_response(
response: tritonserver._api._response.InferenceResponse,
backend: str,
-) -> Optional[CompletionUsage]:
+ request_type: RequestKind,
+) -> Optional[CompletionUsage | EmbeddingUsage]:
"""
Extracts token usage statistics from a Triton inference response.
"""
@@ -293,13 +331,19 @@ def _get_usage_from_response(
)
completion_tokens = completion_tokens_ptr[0]
- if prompt_tokens is not None and completion_tokens is not None:
- total_tokens = prompt_tokens + completion_tokens
- return CompletionUsage(
- prompt_tokens=prompt_tokens,
- completion_tokens=completion_tokens,
- total_tokens=total_tokens,
- )
+ if prompt_tokens is not None:
+ if request_type == RequestKind.GENERATION and completion_tokens is not None:
+ total_tokens = prompt_tokens + completion_tokens
+ return CompletionUsage(
+ prompt_tokens=prompt_tokens,
+ completion_tokens=completion_tokens,
+ total_tokens=total_tokens,
+ )
+ elif request_type == RequestKind.EMBEDDING:
+ return EmbeddingUsage(
+ prompt_tokens=prompt_tokens,
+ total_tokens=prompt_tokens,
+ )
return None
diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py
index 9b73169ade..abe452a595 100644
--- a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py
+++ b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py
@@ -30,7 +30,11 @@
# Mapping of API to their corresponding HTTP endpoints
ENDPOINT_MAPPING = {
- "inference": ["POST /v1/chat/completions", "POST /v1/completions"],
+ "inference": [
+ "POST /v1/chat/completions",
+ "POST /v1/completions",
+ "POST /v1/embeddings",
+ ],
"model-repository": ["GET /v1/models"],
"metrics": ["GET /metrics"],
"health": ["GET /health/ready"],
diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py
new file mode 100644
index 0000000000..84f2604d21
--- /dev/null
+++ b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py
@@ -0,0 +1,50 @@
+# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of NVIDIA CORPORATION nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
+from schemas.openai import CreateEmbeddingRequest, CreateEmbeddingResponse
+
+router = APIRouter()
+
+
+@router.post(
+ "/v1/embeddings", response_model=CreateEmbeddingResponse, tags=["Embeddings"]
+)
+async def create_embedding(
+ request: CreateEmbeddingRequest, raw_request: Request
+) -> CreateEmbeddingResponse | StreamingResponse:
+ """
+ Creates embedding for the provided input text.
+ """
+ if not raw_request.app.engine:
+ raise HTTPException(status_code=500, detail="No attached inference engine")
+
+ try:
+ response = await raw_request.app.engine.embedding(request)
+ return response
+ except Exception as e:
+ raise HTTPException(status_code=400, detail=f"{e}")
diff --git a/python/openai/openai_frontend/frontend/fastapi_frontend.py b/python/openai/openai_frontend/frontend/fastapi_frontend.py
index 2234d84208..e5fb01deae 100644
--- a/python/openai/openai_frontend/frontend/fastapi_frontend.py
+++ b/python/openai/openai_frontend/frontend/fastapi_frontend.py
@@ -34,7 +34,13 @@
APIRestrictionMiddleware,
RestrictedFeatures,
)
-from frontend.fastapi.routers import chat, completions, models, observability
+from frontend.fastapi.routers import (
+ chat,
+ completions,
+ embeddings,
+ models,
+ observability,
+)
from frontend.frontend import OpenAIFrontend
@@ -97,6 +103,7 @@ def _create_app(self):
app.include_router(models.router)
app.include_router(completions.router)
app.include_router(chat.router)
+ app.include_router(embeddings.router)
# NOTE: For debugging purposes, should generally be restricted or removed
self._add_cors_middleware(app)
diff --git a/python/openai/openai_frontend/schemas/openai.py b/python/openai/openai_frontend/schemas/openai.py
index f13296163d..81ff6e93b3 100644
--- a/python/openai/openai_frontend/schemas/openai.py
+++ b/python/openai/openai_frontend/schemas/openai.py
@@ -31,7 +31,7 @@
from __future__ import annotations
from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel, confloat, conint
@@ -601,10 +601,7 @@ class Model(BaseModel):
owned_by: str = Field(..., description="The organization that owns the model.")
-class CompletionUsage(BaseModel):
- completion_tokens: int = Field(
- ..., description="Number of tokens in the generated completion."
- )
+class BaseUsage(BaseModel):
prompt_tokens: int = Field(..., description="Number of tokens in the prompt.")
total_tokens: int = Field(
...,
@@ -612,6 +609,16 @@ class CompletionUsage(BaseModel):
)
+class EmbeddingUsage(BaseUsage):
+ pass
+
+
+class CompletionUsage(BaseUsage):
+ completion_tokens: int = Field(
+ ..., description="Number of tokens in the generated completion."
+ )
+
+
class Event(Enum):
error = "error"
@@ -940,3 +947,69 @@ class ObjectType:
text_completion = Object1.text_completion
chat_completion_chunk = Object4.chat_completion_chunk
chat_completion = Object2.chat_completion
+
+
+class EmbeddingObject(BaseModel):
+ model_config: ConfigDict = ConfigDict(extra="forbid")
+
+ object: Literal["embedding"] = Field(
+ description="The object type, which is always 'embedding'.",
+ )
+ embedding: Union[List[float], str] = Field(
+ ...,
+ description="The embedding vector, which is a list of floats or a base64-encoded string.",
+ )
+ index: int = Field(
+ ...,
+ description="The index of the embedding in the list of embeddings.",
+ )
+
+
+class CreateEmbeddingRequest(BaseModel):
+ # Explicitly return errors for unknown fields.
+ model_config: ConfigDict = ConfigDict(extra="forbid")
+
+ input: Union[str, List[int]] = Field(
+ ...,
+ description="Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.",
+ min_length=1,
+ examples=["The food was delicious and the waiter..."],
+ )
+ model: Union[str, Model2] = Field(
+ ...,
+ description="ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API.",
+ examples=["text-embedding-ada-002"],
+ )
+ dimensions: Optional[int] = Field(
+ None,
+ description="The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.",
+ )
+ encoding_format: Optional[Literal["float", "base64"]] = Field(
+ "float",
+ description="The format to return the embeddings in.",
+ )
+ user: Optional[str] = Field(
+ None,
+ description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n",
+ examples=["user-1234"],
+ )
+
+
+class CreateEmbeddingResponse(BaseModel):
+ model_config: ConfigDict = ConfigDict(extra="forbid")
+
+ object: Literal["list"] = Field(
+ description="The object type, which is always 'list'.",
+ )
+ data: List[EmbeddingObject] = Field(
+ ...,
+ description="The list of embeddings.",
+ )
+ model: Union[str, Model2] = Field(
+ ...,
+ description="The model used to generate the embeddings.",
+ )
+ usage: Optional[EmbeddingUsage] = Field(
+ ...,
+ description="The usage for the request.",
+ )
diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py
index 50ba0de4ed..5c269a78c1 100644
--- a/python/openai/tests/conftest.py
+++ b/python/openai/tests/conftest.py
@@ -67,41 +67,80 @@ def infer_test_model_repository(backend, tool_call_parser):
return model_repository
-# TODO: Refactor away from global variables
-TEST_MODEL = os.environ.get("TEST_MODEL")
-TEST_BACKEND = os.environ.get("TEST_BACKEND")
-TEST_MODEL_REPOSITORY = os.environ.get("TEST_MODEL_REPOSITORY")
+### FIXTURES - Refactored from global variables ###
-TEST_TOKENIZER = os.environ.get(
- "TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct"
-)
-TEST_TOOL_CALL_PARSER = os.environ.get("TEST_TOOL_CALL_PARSER", "llama3")
-TEST_PROMPT = "What is machine learning?"
-TEST_MESSAGES = [{"role": "user", "content": TEST_PROMPT}]
-if not TEST_BACKEND or not TEST_MODEL:
- TEST_BACKEND, TEST_MODEL = infer_test_environment(TEST_TOOL_CALL_PARSER)
+@pytest.fixture(scope="session")
+def tool_call_parser():
+ return os.environ.get("TEST_TOOL_CALL_PARSER", "llama3")
-if not TEST_MODEL_REPOSITORY:
- TEST_MODEL_REPOSITORY = infer_test_model_repository(
- TEST_BACKEND, TEST_TOOL_CALL_PARSER
- )
+
+@pytest.fixture(scope="session")
+def backend(tool_call_parser):
+ env_backend = os.environ.get("TEST_BACKEND")
+ env_model = os.environ.get("TEST_MODEL")
+
+ if not env_backend or not env_model:
+ inferred_backend, _ = infer_test_environment(tool_call_parser)
+ return inferred_backend
+ return env_backend
+
+
+@pytest.fixture(scope="session")
+def model(tool_call_parser):
+ env_model = os.environ.get("TEST_MODEL")
+
+ if not env_model:
+ _, inferred_model = infer_test_environment(tool_call_parser)
+ return inferred_model
+ return env_model
+
+
+@pytest.fixture(scope="session")
+def model_repository(backend, tool_call_parser):
+ env_repo = os.environ.get("TEST_MODEL_REPOSITORY")
+
+ if env_repo:
+ return env_repo
+ return infer_test_model_repository(backend, tool_call_parser)
+
+
+@pytest.fixture(scope="session")
+def tokenizer_model():
+ return os.environ.get("TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct")
+
+
+@pytest.fixture(scope="session")
+def prompt():
+ return "What is machine learning?"
+
+
+@pytest.fixture(scope="session")
+def messages(prompt):
+ return [{"role": "user", "content": prompt}]
+
+
+@pytest.fixture(scope="session")
+def input(prompt):
+ return prompt
# NOTE: OpenAI client requires actual server running, and won't work
# with the FastAPI TestClient. Run the server at module scope to run
# only once for all the tests below.
@pytest.fixture(scope="module")
-def server():
+def server(
+ model_repository: str, tokenizer_model: str, backend: str, tool_call_parser: str
+):
args = [
"--model-repository",
- TEST_MODEL_REPOSITORY,
+ model_repository,
"--tokenizer",
- TEST_TOKENIZER,
+ tokenizer_model,
"--backend",
- TEST_BACKEND,
+ backend,
"--tool-call-parser",
- TEST_TOOL_CALL_PARSER,
+ tool_call_parser,
]
# TODO: Incorporate kserve frontend binding smoke tests to catch any
# breakage with default values or slight cli arg variations
@@ -117,47 +156,17 @@ def server():
# with arbitrary clients - you must use the TestClient returned to interact with
# the "server" when "starting the server" via TestClient.
@pytest.fixture(scope="class")
-def fastapi_client_class_scope():
- server = setup_server(model_repository=TEST_MODEL_REPOSITORY)
- app = setup_fastapi_app(
- tokenizer=TEST_TOKENIZER, server=server, backend=TEST_BACKEND
- )
+def fastapi_client_class_scope(
+ model_repository: str, tokenizer_model: str, backend: str
+):
+ server = setup_server(model_repository=model_repository)
+ app = setup_fastapi_app(tokenizer=tokenizer_model, server=server, backend=backend)
with TestClient(app) as test_client:
yield test_client
server.stop()
-@pytest.fixture(scope="module")
-def model_repository():
- return TEST_MODEL_REPOSITORY
-
-
-@pytest.fixture(scope="module")
-def model():
- return TEST_MODEL
-
-
-@pytest.fixture(scope="module")
-def backend():
- return TEST_BACKEND
-
-
-@pytest.fixture(scope="module")
-def tokenizer_model():
- return TEST_TOKENIZER
-
-
-@pytest.fixture(scope="module")
-def prompt():
- return TEST_PROMPT
-
-
-@pytest.fixture(scope="module")
-def messages():
- return TEST_MESSAGES
-
-
# FIXME: In TRTLLM tests, the in-process Triton server for the FastAPI app
# does not automatically release GPU memory, even after calling stop().
# The memory is only released when the entire pytest process exits.
diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py
index 5402be451d..565146f7c7 100644
--- a/python/openai/tests/test_chat_completions.py
+++ b/python/openai/tests/test_chat_completions.py
@@ -35,7 +35,6 @@
from tests.utils import setup_fastapi_app, setup_server
-@pytest.mark.fastapi
class TestChatCompletions:
@pytest.fixture(scope="class")
def client(self, fastapi_client_class_scope):
@@ -564,7 +563,6 @@ def test_usage_response(
# For tests that won't use the same pytest fixture for server startup across
# the whole class test suite.
-@pytest.mark.fastapi
class TestChatCompletionsTokenizers:
# Re-use a single Triton server for different frontend configurations
@pytest.fixture(scope="class")
diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py
index 9ec3ffe7f7..31eb2f8ae6 100644
--- a/python/openai/tests/test_completions.py
+++ b/python/openai/tests/test_completions.py
@@ -29,7 +29,6 @@
import pytest
-@pytest.mark.fastapi
class TestCompletions:
@pytest.fixture(scope="class")
def client(self, fastapi_client_class_scope):
diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py
new file mode 100644
index 0000000000..b2a21b78e9
--- /dev/null
+++ b/python/openai/tests/test_embeddings.py
@@ -0,0 +1,609 @@
+# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of NVIDIA CORPORATION nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import base64
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+# Results on A6000 GPU. The results vary slightly across GPU models.
+EMBEDDING_OUTPUT_FLOAT = [
+ -0.1914404183626175,
+ 0.4000193178653717,
+ 0.058502197265625,
+ 0.18909454345703125,
+ -0.4690297544002533,
+ 0.004936536308377981,
+ 0.45893096923828125,
+ -0.31141534447669983,
+ 0.18299102783203125,
+ -0.4907582700252533,
+ 0.6920369267463684,
+ -0.001537322998046875,
+ 0.1219015121459961,
+ -0.11682561784982681,
+ 0.02811431884765625,
+ -0.5207672119140625,
+ 0.4574941098690033,
+ -0.31097412109375,
+ 0.13371849060058594,
+ -0.4693959653377533,
+ -0.2766602337360382,
+ 0.029005050659179688,
+ -0.13730454444885254,
+ -0.18662643432617188,
+ 0.0063533782958984375,
+ 0.16905848681926727,
+ 0.1612701416015625,
+ 0.08376502990722656,
+ -0.09822845458984375,
+ -0.012738545425236225,
+ -0.16643650829792023,
+ -0.01901753805577755,
+ -0.0503285713493824,
+ 0.03994830325245857,
+ -0.3373819887638092,
+ -0.0188166294246912,
+ 0.374481201171875,
+ -0.4371846616268158,
+ 0.22470474243164062,
+ -0.011973063461482525,
+ 0.13784568011760712,
+ -0.1484222412109375,
+ 0.19347667694091797,
+ 0.11848513036966324,
+ 0.09258091449737549,
+ -0.2887814939022064,
+ -0.2301533967256546,
+ -0.12088584899902344,
+ 0.1941477507352829,
+ -0.05228869244456291,
+ -0.2508443295955658,
+ 0.15698719024658203,
+ -0.19024403393268585,
+ -0.4728952944278717,
+ 0.336700439453125,
+ 0.11721674352884293,
+ 0.24498240649700165,
+ -0.4826914370059967,
+ -0.119984470307827,
+ 0.1008249893784523,
+ -0.0983428955078125,
+ -0.0059178671799600124,
+ 0.2341969758272171,
+ 0.2725118100643158,
+ 0.2384185791015625,
+ -0.30748113989830017,
+ -0.17387008666992188,
+ -0.44342041015625,
+ -0.15537135303020477,
+ 0.27348455786705017,
+ 0.1131540909409523,
+ -0.23855717480182648,
+ 0.2574056088924408,
+ -0.12090619653463364,
+ -0.14412720501422882,
+ -0.2408244013786316,
+ 0.0021938085556030273,
+ -0.39945730566978455,
+ -0.555908203125,
+ 0.0760548934340477,
+ -0.1530914306640625,
+ -0.40975189208984375,
+ -0.2091045379638672,
+ 0.20317332446575165,
+ -0.20295588672161102,
+ -0.3643442690372467,
+ 0.05287488177418709,
+ -0.24874623119831085,
+ 0.11500009149312973,
+ 0.1661122590303421,
+ 0.26618704199790955,
+ -0.22980372607707977,
+ -0.202911376953125,
+ -0.2738393247127533,
+ 0.20629756152629852,
+ -0.24571101367473602,
+ -0.1486002653837204,
+ -0.12444128841161728,
+ 0.27539315819740295,
+ 0.41679826378822327,
+ -0.01199467945843935,
+ 0.1778361052274704,
+ -0.15123574435710907,
+ -0.0391184501349926,
+ -0.035979270935058594,
+ 0.11838880926370621,
+ -0.07832065969705582,
+ 0.15302227437496185,
+ -0.11540285497903824,
+ -0.008619308471679688,
+ 0.011735956184566021,
+ 0.41825103759765625,
+ 0.1798756867647171,
+ 0.0468953438103199,
+ -0.31410470604896545,
+ -0.28439536690711975,
+ 0.028476715087890625,
+ -0.18972015380859375,
+ -0.1492512971162796,
+ 0.23354721069335938,
+ 0.2631734311580658,
+ 0.3009694516658783,
+ -0.31204381585121155,
+ 0.17155838012695312,
+ -0.6126009821891785,
+ -0.16471035778522491,
+ 0.7154337763786316,
+ 0.0,
+ -0.3936564028263092,
+ -0.15255196392536163,
+ 0.24118296802043915,
+ -0.13930638134479523,
+ 0.6811599731445312,
+ 0.135009765625,
+ -0.18750762939453125,
+ 0.26521047949790955,
+ -0.1257190704345703,
+ 0.0532684326171875,
+ 0.25982680916786194,
+ -0.3410797119140625,
+ -0.189666748046875,
+ 0.016697248443961143,
+ 0.1474812775850296,
+ 0.085713230073452,
+ -0.0862935408949852,
+ 0.521209716796875,
+ 0.3840688169002533,
+ 0.04953320696949959,
+ -0.0478159599006176,
+ -0.3888498842716217,
+ 0.3243462145328522,
+ 0.03093973733484745,
+ -0.3594563901424408,
+ 0.16615693271160126,
+ -0.07209650427103043,
+ 0.049218177795410156,
+ 0.14628247916698456,
+ -0.10561561584472656,
+ 0.1696879118680954,
+ 0.1195220947265625,
+ 0.0140139264985919,
+ 0.08987680822610855,
+ 0.02198282815515995,
+ -0.06835142523050308,
+ -0.09100532531738281,
+ -0.3970082700252533,
+ -0.20552189648151398,
+ -0.0871327742934227,
+ -0.008806228637695312,
+ 0.10437265783548355,
+ 0.2754974365234375,
+ 0.2630208432674408,
+ -0.67779541015625,
+ 0.32654380798339844,
+ -0.4008077085018158,
+ 0.2785542905330658,
+ 0.16632080078125,
+ -0.0709940567612648,
+ -0.1678619384765625,
+ -0.11333879083395004,
+ 0.5577189326286316,
+ 0.3165779113769531,
+ -0.2243397980928421,
+ 0.08053144067525864,
+ 0.1904652863740921,
+ 0.22478973865509033,
+ 0.11852264404296875,
+ -0.2071024626493454,
+ 0.2380015105009079,
+ 0.4622955322265625,
+ 0.1029459610581398,
+ -0.30094656348228455,
+ 0.0351104736328125,
+ -0.09827486425638199,
+ 0.0018183389911428094,
+ 0.07406362146139145,
+ 0.18090057373046875,
+ 0.2231648713350296,
+ -0.1001536026597023,
+ 0.06609535217285156,
+ 0.0055376687087118626,
+ -0.02939859963953495,
+ -0.17679977416992188,
+ 0.2300567626953125,
+ -0.232757568359375,
+ -0.1863892823457718,
+ 0.14040501415729523,
+ -0.21081669628620148,
+ 0.4772237241268158,
+ 0.00708770751953125,
+ 0.25393548607826233,
+ -0.12926609814167023,
+ -0.21408335864543915,
+ 0.43414306640625,
+ -0.16021983325481415,
+ -0.6590754389762878,
+ 0.383026123046875,
+ 0.4894002377986908,
+ -0.5350291132926941,
+ 0.1563262939453125,
+ 0.4013887941837311,
+ -0.1429697722196579,
+ -0.1266673356294632,
+ 0.0,
+ -0.12781651318073273,
+ 0.5082905888557434,
+ -0.4895477294921875,
+ 0.05857785418629646,
+ -0.01038360595703125,
+ -0.4025942385196686,
+ -0.6376139521598816,
+ -0.27256616950035095,
+ -0.2183430939912796,
+ 0.13019943237304688,
+ -0.2378387451171875,
+ -0.12579791247844696,
+ 0.23233287036418915,
+ -0.1948690414428711,
+ -0.10780048370361328,
+ 0.4768002927303314,
+ 0.2761942446231842,
+ 0.09968694299459457,
+ -0.07807016372680664,
+ 0.18632762134075165,
+ -0.014780680648982525,
+ 0.18301646411418915,
+ 0.10943603515625,
+ 0.45223236083984375,
+ -0.24634425342082977,
+ 0.5127970576286316,
+ 0.15272267162799835,
+ 0.26901498436927795,
+ -0.8670451045036316,
+ -0.20471616089344025,
+ 0.3934173583984375,
+ -0.22558848559856415,
+ 0.14676158130168915,
+ -0.16282017529010773,
+ 0.0047810873948037624,
+ 0.49467912316322327,
+ -0.1040293350815773,
+ -0.13565094769001007,
+ -0.05704273656010628,
+ 0.2030487060546875,
+ 0.27226924896240234,
+ -0.16900062561035156,
+ 0.06879997253417969,
+ 0.44347524642944336,
+ 0.08619359880685806,
+ -0.1269734650850296,
+ -0.05267079547047615,
+ -0.3465728759765625,
+ 0.1846415251493454,
+ -0.0655873641371727,
+ 0.027518590912222862,
+ -0.06689834594726562,
+ -0.13316090404987335,
+ -0.3649355471134186,
+ -0.0573628731071949,
+ 0.030780792236328125,
+ 0.2462870329618454,
+ -0.0250523891299963,
+ 0.08964482694864273,
+ -0.34076571464538574,
+ -0.3342704772949219,
+ -0.000331878662109375,
+ 0.25020280480384827,
+ 0.34731578826904297,
+ 0.4081510007381439,
+ 0.0661773681640625,
+ 0.14612038433551788,
+ -0.37111154198646545,
+ -0.17901070415973663,
+ 0.0565798282623291,
+ -0.1689503937959671,
+ 0.311676025390625,
+ 0.06296539306640625,
+ 0.11648496240377426,
+ -0.16365115344524384,
+ -0.011795361526310444,
+ -0.4601001739501953,
+ 0.13840866088867188,
+ 0.1115519180893898,
+ -0.3645426332950592,
+ -0.182403564453125,
+ -0.20782725512981415,
+ -0.004481792449951172,
+ 0.0870104655623436,
+ -0.11704126745462418,
+ 0.34148290753364563,
+ 0.17841561138629913,
+ -0.2754109799861908,
+ -0.0867462158203125,
+ 0.09910837560892105,
+ -0.14540545642375946,
+ -0.10996246337890625,
+ -0.10946687310934067,
+ 0.023001352325081825,
+ 0.11987527459859848,
+ -5.960464477539063e-8,
+ 0.3316993713378906,
+ -0.025622526183724403,
+ -0.28015899658203125,
+ 0.34741735458374023,
+ 0.04091135784983635,
+ -0.34874120354652405,
+ 0.22758229076862335,
+ -0.042999267578125,
+ 0.0382130928337574,
+ 0.5654922127723694,
+ -0.9378255009651184,
+ 0.17114512622356415,
+ 0.13035202026367188,
+ 0.4369252622127533,
+ 0.0897369384765625,
+ 0.19928233325481415,
+ 0.33091607689857483,
+ -0.10624822229146957,
+ -0.2845611572265625,
+ 0.2822163999080658,
+ 0.1722426414489746,
+ 0.2111460417509079,
+ -0.1069692000746727,
+ -0.3496347963809967,
+ 0.15000660717487335,
+ 0.014147520065307617,
+ -0.36633554100990295,
+ 0.23989041149616241,
+ -0.06397350877523422,
+ 0.2501627504825592,
+ 0.04016287997364998,
+ -0.3789469301700592,
+ -0.4247843325138092,
+ 0.1515035629272461,
+ 0.36554718017578125,
+ 0.057392120361328125,
+ -0.3492482602596283,
+ -0.45532989501953125,
+ 0.4090474545955658,
+ -0.3914286196231842,
+ -0.4888407289981842,
+ 0.4746551513671875,
+ -0.6188761591911316,
+ -0.018857955932617188,
+ 0.02373504638671875,
+ 0.22691090404987335,
+ -0.07608286291360855,
+ 0.5331514477729797,
+ -0.27182260155677795,
+ 0.2309315949678421,
+ -0.1824493408203125,
+ 0.12648265063762665,
+ 0.2586142122745514,
+ -0.07648912817239761,
+ 0.2318166047334671,
+ -0.5225245356559753,
+ 0.133880615234375,
+ -0.010974247939884663,
+ 0.09001413732767105,
+ 0.2562611997127533,
+ 0.19260406494140625,
+ 0.4470011293888092,
+ -0.1636505126953125,
+ -0.3675130307674408,
+]
+
+
+@pytest.mark.skipif(
+ os.environ.get("IMAGE_KIND") == "TRTLLM",
+ reason="TRT-LLM backend does not support embedding requests",
+)
+class TestEmbeddings:
+ @pytest.fixture(scope="class")
+ def client(self, fastapi_client_class_scope):
+ yield fastapi_client_class_scope
+
+ @pytest.fixture(scope="class")
+ def model(self):
+ # Override with embeddings-specific model
+ return "all-MiniLM-L6-v2"
+
+ @pytest.fixture(scope="class")
+ def tokenizer_model(self):
+ return None
+
+ @pytest.fixture(scope="class")
+ def model_repository(self):
+ # Override with embeddings-specific repository
+ return str(Path(__file__).parent / "vllm_embedding_models")
+
+ @pytest.fixture(scope="class")
+ def input(self):
+ return "The food was delicious and the waiter..."
+
+ def _check_embedding_response(
+ self, response, model, dims=len(EMBEDDING_OUTPUT_FLOAT), encoding_format="float"
+ ):
+ assert response.status_code == 200, response.json()
+ embedding = response.json()["data"][0]["embedding"]
+ assert embedding is not None
+ if encoding_format == "base64":
+ embedding = np.frombuffer(base64.b64decode(embedding), dtype=np.float32)
+
+ # The results vary slightly across GPU models
+ result = np.allclose(
+ EMBEDDING_OUTPUT_FLOAT[:dims], embedding, rtol=0, atol=1e-3
+ )
+ assert (
+ result
+ ), f"Embeddings do not match expected output\nExpect {EMBEDDING_OUTPUT_FLOAT[:dims]},\ngot{embedding}"
+
+ assert response.json()["data"][0]["object"] == "embedding"
+ assert response.json()["data"][0]["index"] == 0
+ assert response.json()["model"] == model
+
+ usage = response.json().get("usage")
+ assert usage is not None
+ assert usage["prompt_tokens"] == 12
+ assert usage["total_tokens"] == 12
+
+ @pytest.mark.parametrize(
+ "input",
+ [
+ "The food was delicious and the waiter...",
+ [101, 1996, 2833, 2001, 12090, 1998, 1996, 15610, 1012, 1012, 1012, 102],
+ ],
+ )
+ def test_embeddings_defaults(self, client, model: str, input: str):
+ response = client.post(
+ "/v1/embeddings",
+ json={"model": model, "input": input},
+ )
+
+ self._check_embedding_response(response, model)
+
+ # FIXME: Python model cannot unload gracefully if raise error.
+ # def test_chat_completions_defaults(
+ # self, client, model: str, messages: List[dict], backend: str
+ # ):
+ # response = client.post(
+ # "/v1/chat/completions",
+ # json={"model": model, "messages": messages},
+ # )
+
+ # assert response.status_code == 400
+ # assert "does not support" in response.json()["detail"]
+
+ @pytest.mark.parametrize(
+ "param_key, param_value",
+ [
+ ("dimensions", [10]),
+ ("encoding_format", "invalid"),
+ ("encoding_format", 0),
+ ],
+ )
+ def test_embeddings_invalid_parameters(
+ self, client, param_key, param_value, model: str, input: str
+ ):
+ response = client.post(
+ "/v1/embeddings",
+ json={
+ "model": model,
+ "input": input,
+ param_key: param_value,
+ },
+ )
+
+ # Assert schema validation error
+ assert response.status_code == 422, response.json()
+
+ @pytest.mark.parametrize("dimensions", [0, 10, 100, -1])
+ @pytest.mark.parametrize("encoding_format", ["float", "base64"])
+ def test_embeddings_parameters(
+ self, client, dimensions, encoding_format, model: str, input: str
+ ):
+ response = client.post(
+ "/v1/embeddings",
+ json={
+ "model": model,
+ "input": input,
+ "dimensions": dimensions,
+ "encoding_format": encoding_format,
+ },
+ )
+
+ self._check_embedding_response(
+ response, model, dims=dimensions, encoding_format=encoding_format
+ )
+
+ def test_embeddings_empty_request(self, client):
+ response = client.post("/v1/embeddings", json={})
+ assert response.status_code == 422
+ assert response.json()["detail"][0]["msg"] == "Field required"
+
+ def test_embeddings_no_model(self, client, input: str):
+ response = client.post("/v1/embeddings", json={"input": input})
+ assert response.status_code == 422
+ assert response.json()["detail"][0]["msg"] == "Field required"
+
+ @pytest.mark.parametrize(
+ "model, error_code",
+ [
+ ("", 400),
+ (123, 422),
+ ("Invalid", 400),
+ (None, 422),
+ ],
+ )
+ def test_embeddings_invalid_model(self, client, model: str, input, error_code: int):
+ print("Model:", model)
+ # Message validation requires min_length of 1
+ response = client.post("/v1/embeddings", json={"model": model, "input": input})
+ assert response.status_code == error_code
+ if error_code == 400:
+ assert response.json()["detail"] == f"Unknown model: {model}"
+ else:
+ assert (
+ response.json()["detail"][0]["msg"] == "Input should be a valid string"
+ )
+
+ def test_embeddings_no_input(self, client, model: str):
+ response = client.post("/v1/embeddings", json={"model": model})
+ assert response.status_code == 422
+
+ @pytest.mark.parametrize(
+ "input",
+ [
+ "",
+ [],
+ ],
+ )
+ def test_embeddings_empty_input(self, client, model: str, input):
+ # Message validation requires min_length of 1
+ response = client.post("/v1/embeddings", json={"model": model, "input": input})
+ assert response.status_code == 422
+ assert (
+ response.json()["detail"][0]["msg"]
+ == "Value should have at least 1 item after validation, not 0"
+ )
+
+ @pytest.mark.parametrize(
+ "input",
+ [
+ 123,
+ 1.5,
+ 0,
+ None,
+ ],
+ )
+ def test_embeddings_invalid_input(self, client, model: str, input):
+ # Message validation requires min_length of 1
+ response = client.post("/v1/embeddings", json={"model": model, "input": input})
+ assert response.status_code == 422
+ assert response.json()["detail"][0]["msg"] == "Input should be a valid string"
diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json
new file mode 100644
index 0000000000..2ad058c2e5
--- /dev/null
+++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json
@@ -0,0 +1 @@
+{"model": "sentence-transformers/all-MiniLM-L6-v2", "gpu_memory_utilization": 0.5}
diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt
new file mode 100644
index 0000000000..39b3c48edb
--- /dev/null
+++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt
@@ -0,0 +1,28 @@
+# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+# * Neither the name of NVIDIA CORPORATION nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+backend: "vllm"
+instance_group [{kind: KIND_MODEL}]