diff --git a/python/openai/README.md b/python/openai/README.md index 572373390f..bcabf5e969 100644 --- a/python/openai/README.md +++ b/python/openai/README.md @@ -301,6 +301,98 @@ See the [vLLM documentation](https://github.com/triton-inference-server/vllm_backend/blob/main/docs/llama_multi_lora_tutorial.md) on how to serve a model with LoRA adapters. +### Embedding Models +Currently, OpenAI-Compatible Frontend supports loading embedding models and embeddings endpoints via vLLM backend. Check [vLLM supported models](https://docs.vllm.ai/en/latest/models/supported_models.html#embedding) for all supported embedding models from vLLM. + +1. Launch the container and install dependencies: + - Mounts the `~/.huggingface/cache` for re-use of downloaded models across runs, containers, etc. + - Sets the [`HF_TOKEN`](https://huggingface.co/docs/huggingface_hub/en/package_reference/environment_variables#hftoken) environment variable to + access gated models, make sure this is set in your local environment if needed. + +```bash +docker run -it --net=host --gpus all --rm \ + -v ${HOME}/.cache/huggingface:/root/.cache/huggingface \ + -e HF_TOKEN \ + nvcr.io/nvidia/tritonserver:25.10-vllm-python-py3 +``` + +2. Launch the OpenAI-compatible Triton Inference Server: +```bash +cd /opt/tritonserver/python/openai + +# NOTE: Embeddings endpoint does not require "--tokenizer" +python3 openai_frontend/main.py --model-repository tests/vllm_embedding_models +``` + +
+Example output + +``` +... ++------------------+---------+--------+ +| Model | Version | Status | ++------------------+---------+--------+ +| all-MiniLM-L6-v2 | 1 | READY | <- Correct Model Loaded in Triton ++------------------+---------+--------+ +... +Found model: name='all-MiniLM-L6-v2', backend='vllm' +[WARNING] Adding CORS for the following origins: ['http://localhost'] +INFO: Started server process [133] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://0.0.0.0:9000 (Press CTRL+C to quit) <- OpenAI Frontend Started Successfully +``` + +
+ +3. Send a `/v1/embeddings` request: + - Note the use of `jq` is optional, but provides a nicely formatted output for JSON responses. +```bash +MODEL="all-MiniLM-L6-v2" +curl -s http://localhost:9000/v1/embeddings \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "'${MODEL}'", + "input": "The food was delicious and the waiter...", + "dimensions": 10, + "encoding_format": "float" + }' | jq +``` + +
+Example output + +```json +{ + "object": "list", + "data": [ + { + "object": "embedding", + "embedding": [ + -0.1914404183626175, + 0.4000193178653717, + 0.058502197265625, + 0.18909454345703125, + -0.4690297544002533, + 0.004936536308377981, + 0.45893096923828125, + -0.31141534447669983, + 0.18299102783203125, + -0.4907582700252533 + ], + "index": 0 + } + ], + "model": "all-MiniLM-L6-v2", + "usage": { + "prompt_tokens": 12, + "total_tokens": 12 + } +} +``` + +
+ ## TensorRT-LLM 0. Prepare your model repository for a TensorRT-LLM model, build the engine, etc. You can try any of the following options: @@ -655,6 +747,8 @@ Use the `--openai-restricted-api` command-line argument to configure endpoint re - **inference**: Chat completions and text completions endpoints - `POST /v1/chat/completions` - `POST /v1/completions` + - **embedding**: Embedding endpoint + - `POST /v1/embeddings` - **model-repository**: Model listing and information endpoints - `GET /v1/models` - `GET /v1/models/{model_name}` diff --git a/python/openai/openai_frontend/engine/engine.py b/python/openai/openai_frontend/engine/engine.py index 9c90dec25e..2dfeafb1db 100644 --- a/python/openai/openai_frontend/engine/engine.py +++ b/python/openai/openai_frontend/engine/engine.py @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -34,6 +34,8 @@ CreateChatCompletionResponse, CreateCompletionRequest, CreateCompletionResponse, + CreateEmbeddingRequest, + CreateEmbeddingResponse, Model, ) @@ -92,3 +94,9 @@ def completion( If request.stream is False, this returns a CreateCompletionResponse. """ pass + + def embedding(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse: + """ + Returns a CreateEmbeddingResponse. + """ + pass diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 499cc623e7..f0d184cca2 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -27,6 +27,7 @@ from __future__ import annotations +import base64 import json import time import uuid @@ -38,18 +39,24 @@ Callable, Dict, List, + Literal, Optional, Tuple, + Union, ) +import numpy as np import tritonserver from engine.engine import LLMEngine from engine.utils.chat import load_chat_template, parse_chat_messages from engine.utils.tokenizer import get_tokenizer from engine.utils.tool_call_parsers import ToolCallParser, ToolParserManager from engine.utils.triton import ( - _create_trtllm_inference_request, - _create_vllm_inference_request, + RequestKind, + _create_trtllm_embedding_request, + _create_trtllm_generate_request, + _create_vllm_embedding_request, + _create_vllm_generate_request, _get_output, _get_usage_from_response, _get_vllm_lora_names, @@ -73,6 +80,9 @@ CreateChatCompletionStreamResponse, CreateCompletionRequest, CreateCompletionResponse, + CreateEmbeddingRequest, + CreateEmbeddingResponse, + EmbeddingObject, FinishReason, Function1, Function2, @@ -97,7 +107,8 @@ class TritonModelMetadata: # Time that model was loaded by Triton create_time: int # Conversion format between OpenAI and Triton requests - request_converter: Callable + inference_request_converter: Callable + embedding_request_converter: Callable class TritonLLMEngine(LLMEngine): @@ -189,7 +200,7 @@ async def chat( # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.request_converter( + metadata.inference_request_converter( metadata.model, prompt, request, lora_name, self.default_max_tokens ) ) @@ -232,7 +243,9 @@ async def chat( backend=metadata.backend, ) - usage = _get_usage_from_response(response, metadata.backend) + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.GENERATION + ) return CreateChatCompletionResponse( id=request_id, @@ -311,7 +324,7 @@ async def completion( # Convert to Triton request format and perform inference responses = metadata.model.async_infer( - metadata.request_converter( + metadata.inference_request_converter( metadata.model, request.prompt, request, @@ -334,7 +347,9 @@ async def completion( response = responses[0] text = _get_output(response) - usage = _get_usage_from_response(response, metadata.backend) + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.GENERATION + ) choice = Choice( finish_reason=FinishReason.stop, @@ -352,6 +367,57 @@ async def completion( usage=usage, ) + async def embedding( + self, request: CreateEmbeddingRequest + ) -> CreateEmbeddingResponse: + # Validate request and convert to Triton format + model_name, _ = self._get_model_and_lora_name(request.model) + metadata = self.model_metadata.get(model_name) + self._validate_embedding_request(request, metadata) + + # Convert to Triton request format and perform inference + responses = metadata.model.async_infer( + metadata.embedding_request_converter( + metadata.model, + request, + ) + ) + + # Response validation with decoupled models in mind + responses = [response async for response in responses] + _validate_triton_responses_non_streaming(responses) + response = responses[0] + + # Extract embedding from response (currently stored as JSON string in text_output) + embedding_json = _get_output(response) + embedding_list = json.loads(embedding_json) + + usage = _get_usage_from_response( + response, metadata.backend, RequestKind.EMBEDDING + ) + + embedding = self._get_embedding(embedding_list, request.encoding_format) + embedding_obj = EmbeddingObject( + embedding=embedding, index=0, object="embedding" + ) + + return CreateEmbeddingResponse( + object="list", + data=[embedding_obj], + model=request.model, + usage=usage, + ) + + @staticmethod + def _get_embedding( + embedding: List[float], encoding_format: Literal["float", "base64"] + ) -> Union[list[float], str]: + if encoding_format == "float": + return embedding + else: + embedding_bytes = np.array(embedding, dtype="float32").tobytes() + return base64.b64encode(embedding_bytes).decode("utf-8") + # TODO: This behavior should be tested further def _get_first_response_role( self, conversation: List[Dict], add_generation_prompt: bool, default_role: str @@ -362,18 +428,24 @@ def _get_first_response_role( return conversation[-1]["role"] # TODO: Expose explicit flag to catch edge cases - def _determine_request_converter(self, backend: str): + def _determine_request_converter(self, backend: str, request_type: RequestKind): # Allow manual override of backend request format if provided by user if self.backend: backend = self.backend # Request conversion from OpenAI format to backend-specific format if backend == "vllm": - return _create_vllm_inference_request + if request_type == RequestKind.GENERATION: + return _create_vllm_generate_request + else: + return _create_vllm_embedding_request # Use TRT-LLM format as default for everything else. This could be # an ensemble, a python or BLS model, a TRT-LLM backend model, etc. - return _create_trtllm_inference_request + if request_type == RequestKind.GENERATION: + return _create_trtllm_generate_request + else: + return _create_trtllm_embedding_request def _get_model_and_lora_name(self, request_model_name: str): if self.lora_separator is None or len(self.lora_separator) == 0: @@ -418,7 +490,12 @@ def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]: tokenizer=self.tokenizer, lora_names=lora_names, create_time=self.create_time, - request_converter=self._determine_request_converter(backend), + inference_request_converter=self._determine_request_converter( + backend, RequestKind.GENERATION + ), + embedding_request_converter=self._determine_request_converter( + backend, RequestKind.EMBEDDING + ), ) model_metadata[name] = metadata @@ -671,8 +748,15 @@ def _validate_chat_request( if not metadata.backend: raise Exception("Unknown backend") - if not metadata.request_converter: - raise Exception(f"Unknown request format for model: {request.model}") + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) if ( metadata.lora_names is not None @@ -807,8 +891,15 @@ def _validate_completion_request( if not metadata.backend: raise Exception("Unknown backend") - if not metadata.request_converter: - raise Exception(f"Unknown request format for model: {request.model}") + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) if ( metadata.lora_names is not None @@ -853,6 +944,32 @@ def _validate_completion_request( "`stream_options.include_usage` is currently only supported for the vLLM backend" ) + def _validate_embedding_request( + self, + request: CreateEmbeddingRequest, + metadata: TritonModelMetadata, + ): + """ + Validates an embedding request to align with currently supported features. + """ + + # Reject missing internal information needed to do inference + if not metadata: + raise Exception(f"Unknown model: {request.model}") + + if not metadata.backend: + raise Exception("Unknown backend") + + if not metadata.inference_request_converter: + raise Exception( + f"Unknown inference request format for model: {request.model}" + ) + + if not metadata.embedding_request_converter: + raise Exception( + f"Unknown embedding request format for model: {request.model}" + ) + def _should_stream_with_auto_tool_parsing( self, request: CreateChatCompletionRequest ): diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 636e58435d..6258c4b9df 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -28,6 +28,7 @@ import os import re from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Iterable, List, Optional, Union @@ -40,10 +41,17 @@ CompletionUsage, CreateChatCompletionRequest, CreateCompletionRequest, + CreateEmbeddingRequest, + EmbeddingUsage, ) -def _create_vllm_inference_request( +class RequestKind(Enum): + GENERATION = 1 + EMBEDDING = 2 + + +def _create_vllm_generate_request( model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest, @@ -128,7 +136,7 @@ def _create_vllm_inference_request( return model.create_request(inputs=inputs) -def _create_trtllm_inference_request( +def _create_trtllm_generate_request( model, prompt, request: CreateChatCompletionRequest | CreateCompletionRequest, @@ -183,6 +191,35 @@ def _create_trtllm_inference_request( return model.create_request(inputs=inputs) +def _create_vllm_embedding_request( + model, + request: CreateEmbeddingRequest, +): + inputs = {} + embedding_request = {} + embedding_request["input"] = request.input + + pooling_params = {} + dims = request.dimensions + if dims is not None: + pooling_params["dimensions"] = [dims] + embedding_request["pooling_params"] = pooling_params + + inputs["embedding_request"] = [json.dumps(embedding_request)] + inputs["return_num_input_tokens"] = np.bool_([True]) + inputs["return_num_output_tokens"] = np.bool_([True]) + return model.create_request(inputs=inputs) + + +def _create_trtllm_embedding_request( + model, + request: CreateEmbeddingRequest, +): + raise Exception( + "TRT-LLM backend and Python backend do not support embedding requests" + ) + + def _construct_string_from_pointer(pointer: int, size: int) -> str: """Constructs a Python string from a C pointer and size.""" @@ -236,7 +273,7 @@ class _StreamingUsageAccumulator: def update(self, response: tritonserver.InferenceResponse): """Extracts usage from a response and updates the token counts.""" - usage = _get_usage_from_response(response, self.backend) + usage = _get_usage_from_response(response, self.backend, RequestKind.GENERATION) if usage: # The prompt_tokens is received with every chunk but should only be set once. if not self._prompt_tokens_set: @@ -262,7 +299,8 @@ def get_final_usage(self) -> Optional[CompletionUsage]: def _get_usage_from_response( response: tritonserver._api._response.InferenceResponse, backend: str, -) -> Optional[CompletionUsage]: + request_type: RequestKind, +) -> Optional[CompletionUsage | EmbeddingUsage]: """ Extracts token usage statistics from a Triton inference response. """ @@ -293,13 +331,19 @@ def _get_usage_from_response( ) completion_tokens = completion_tokens_ptr[0] - if prompt_tokens is not None and completion_tokens is not None: - total_tokens = prompt_tokens + completion_tokens - return CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + if prompt_tokens is not None: + if request_type == RequestKind.GENERATION and completion_tokens is not None: + total_tokens = prompt_tokens + completion_tokens + return CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + elif request_type == RequestKind.EMBEDDING: + return EmbeddingUsage( + prompt_tokens=prompt_tokens, + total_tokens=prompt_tokens, + ) return None diff --git a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py index 9b73169ade..abe452a595 100644 --- a/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py +++ b/python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py @@ -30,7 +30,11 @@ # Mapping of API to their corresponding HTTP endpoints ENDPOINT_MAPPING = { - "inference": ["POST /v1/chat/completions", "POST /v1/completions"], + "inference": [ + "POST /v1/chat/completions", + "POST /v1/completions", + "POST /v1/embeddings", + ], "model-repository": ["GET /v1/models"], "metrics": ["GET /metrics"], "health": ["GET /health/ready"], diff --git a/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py new file mode 100644 index 0000000000..84f2604d21 --- /dev/null +++ b/python/openai/openai_frontend/frontend/fastapi/routers/embeddings.py @@ -0,0 +1,50 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import StreamingResponse +from schemas.openai import CreateEmbeddingRequest, CreateEmbeddingResponse + +router = APIRouter() + + +@router.post( + "/v1/embeddings", response_model=CreateEmbeddingResponse, tags=["Embeddings"] +) +async def create_embedding( + request: CreateEmbeddingRequest, raw_request: Request +) -> CreateEmbeddingResponse | StreamingResponse: + """ + Creates embedding for the provided input text. + """ + if not raw_request.app.engine: + raise HTTPException(status_code=500, detail="No attached inference engine") + + try: + response = await raw_request.app.engine.embedding(request) + return response + except Exception as e: + raise HTTPException(status_code=400, detail=f"{e}") diff --git a/python/openai/openai_frontend/frontend/fastapi_frontend.py b/python/openai/openai_frontend/frontend/fastapi_frontend.py index 2234d84208..e5fb01deae 100644 --- a/python/openai/openai_frontend/frontend/fastapi_frontend.py +++ b/python/openai/openai_frontend/frontend/fastapi_frontend.py @@ -34,7 +34,13 @@ APIRestrictionMiddleware, RestrictedFeatures, ) -from frontend.fastapi.routers import chat, completions, models, observability +from frontend.fastapi.routers import ( + chat, + completions, + embeddings, + models, + observability, +) from frontend.frontend import OpenAIFrontend @@ -97,6 +103,7 @@ def _create_app(self): app.include_router(models.router) app.include_router(completions.router) app.include_router(chat.router) + app.include_router(embeddings.router) # NOTE: For debugging purposes, should generally be restricted or removed self._add_cors_middleware(app) diff --git a/python/openai/openai_frontend/schemas/openai.py b/python/openai/openai_frontend/schemas/openai.py index f13296163d..81ff6e93b3 100644 --- a/python/openai/openai_frontend/schemas/openai.py +++ b/python/openai/openai_frontend/schemas/openai.py @@ -31,7 +31,7 @@ from __future__ import annotations from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel, confloat, conint @@ -601,10 +601,7 @@ class Model(BaseModel): owned_by: str = Field(..., description="The organization that owns the model.") -class CompletionUsage(BaseModel): - completion_tokens: int = Field( - ..., description="Number of tokens in the generated completion." - ) +class BaseUsage(BaseModel): prompt_tokens: int = Field(..., description="Number of tokens in the prompt.") total_tokens: int = Field( ..., @@ -612,6 +609,16 @@ class CompletionUsage(BaseModel): ) +class EmbeddingUsage(BaseUsage): + pass + + +class CompletionUsage(BaseUsage): + completion_tokens: int = Field( + ..., description="Number of tokens in the generated completion." + ) + + class Event(Enum): error = "error" @@ -940,3 +947,69 @@ class ObjectType: text_completion = Object1.text_completion chat_completion_chunk = Object4.chat_completion_chunk chat_completion = Object2.chat_completion + + +class EmbeddingObject(BaseModel): + model_config: ConfigDict = ConfigDict(extra="forbid") + + object: Literal["embedding"] = Field( + description="The object type, which is always 'embedding'.", + ) + embedding: Union[List[float], str] = Field( + ..., + description="The embedding vector, which is a list of floats or a base64-encoded string.", + ) + index: int = Field( + ..., + description="The index of the embedding in the list of embeddings.", + ) + + +class CreateEmbeddingRequest(BaseModel): + # Explicitly return errors for unknown fields. + model_config: ConfigDict = ConfigDict(extra="forbid") + + input: Union[str, List[int]] = Field( + ..., + description="Input text to embed, encoded as a string or array of tokens. To embed multiple inputs in a single request, pass an array of strings or array of token arrays.", + min_length=1, + examples=["The food was delicious and the waiter..."], + ) + model: Union[str, Model2] = Field( + ..., + description="ID of the model to use. See the [model endpoint compatibility](/docs/models/model-endpoint-compatibility) table for details on which models work with the Chat API.", + examples=["text-embedding-ada-002"], + ) + dimensions: Optional[int] = Field( + None, + description="The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.", + ) + encoding_format: Optional[Literal["float", "base64"]] = Field( + "float", + description="The format to return the embeddings in.", + ) + user: Optional[str] = Field( + None, + description="A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. [Learn more](/docs/guides/safety-best-practices/end-user-ids).\n", + examples=["user-1234"], + ) + + +class CreateEmbeddingResponse(BaseModel): + model_config: ConfigDict = ConfigDict(extra="forbid") + + object: Literal["list"] = Field( + description="The object type, which is always 'list'.", + ) + data: List[EmbeddingObject] = Field( + ..., + description="The list of embeddings.", + ) + model: Union[str, Model2] = Field( + ..., + description="The model used to generate the embeddings.", + ) + usage: Optional[EmbeddingUsage] = Field( + ..., + description="The usage for the request.", + ) diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index 50ba0de4ed..5c269a78c1 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -67,41 +67,80 @@ def infer_test_model_repository(backend, tool_call_parser): return model_repository -# TODO: Refactor away from global variables -TEST_MODEL = os.environ.get("TEST_MODEL") -TEST_BACKEND = os.environ.get("TEST_BACKEND") -TEST_MODEL_REPOSITORY = os.environ.get("TEST_MODEL_REPOSITORY") +### FIXTURES - Refactored from global variables ### -TEST_TOKENIZER = os.environ.get( - "TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct" -) -TEST_TOOL_CALL_PARSER = os.environ.get("TEST_TOOL_CALL_PARSER", "llama3") -TEST_PROMPT = "What is machine learning?" -TEST_MESSAGES = [{"role": "user", "content": TEST_PROMPT}] -if not TEST_BACKEND or not TEST_MODEL: - TEST_BACKEND, TEST_MODEL = infer_test_environment(TEST_TOOL_CALL_PARSER) +@pytest.fixture(scope="session") +def tool_call_parser(): + return os.environ.get("TEST_TOOL_CALL_PARSER", "llama3") -if not TEST_MODEL_REPOSITORY: - TEST_MODEL_REPOSITORY = infer_test_model_repository( - TEST_BACKEND, TEST_TOOL_CALL_PARSER - ) + +@pytest.fixture(scope="session") +def backend(tool_call_parser): + env_backend = os.environ.get("TEST_BACKEND") + env_model = os.environ.get("TEST_MODEL") + + if not env_backend or not env_model: + inferred_backend, _ = infer_test_environment(tool_call_parser) + return inferred_backend + return env_backend + + +@pytest.fixture(scope="session") +def model(tool_call_parser): + env_model = os.environ.get("TEST_MODEL") + + if not env_model: + _, inferred_model = infer_test_environment(tool_call_parser) + return inferred_model + return env_model + + +@pytest.fixture(scope="session") +def model_repository(backend, tool_call_parser): + env_repo = os.environ.get("TEST_MODEL_REPOSITORY") + + if env_repo: + return env_repo + return infer_test_model_repository(backend, tool_call_parser) + + +@pytest.fixture(scope="session") +def tokenizer_model(): + return os.environ.get("TEST_TOKENIZER", "meta-llama/Meta-Llama-3.1-8B-Instruct") + + +@pytest.fixture(scope="session") +def prompt(): + return "What is machine learning?" + + +@pytest.fixture(scope="session") +def messages(prompt): + return [{"role": "user", "content": prompt}] + + +@pytest.fixture(scope="session") +def input(prompt): + return prompt # NOTE: OpenAI client requires actual server running, and won't work # with the FastAPI TestClient. Run the server at module scope to run # only once for all the tests below. @pytest.fixture(scope="module") -def server(): +def server( + model_repository: str, tokenizer_model: str, backend: str, tool_call_parser: str +): args = [ "--model-repository", - TEST_MODEL_REPOSITORY, + model_repository, "--tokenizer", - TEST_TOKENIZER, + tokenizer_model, "--backend", - TEST_BACKEND, + backend, "--tool-call-parser", - TEST_TOOL_CALL_PARSER, + tool_call_parser, ] # TODO: Incorporate kserve frontend binding smoke tests to catch any # breakage with default values or slight cli arg variations @@ -117,47 +156,17 @@ def server(): # with arbitrary clients - you must use the TestClient returned to interact with # the "server" when "starting the server" via TestClient. @pytest.fixture(scope="class") -def fastapi_client_class_scope(): - server = setup_server(model_repository=TEST_MODEL_REPOSITORY) - app = setup_fastapi_app( - tokenizer=TEST_TOKENIZER, server=server, backend=TEST_BACKEND - ) +def fastapi_client_class_scope( + model_repository: str, tokenizer_model: str, backend: str +): + server = setup_server(model_repository=model_repository) + app = setup_fastapi_app(tokenizer=tokenizer_model, server=server, backend=backend) with TestClient(app) as test_client: yield test_client server.stop() -@pytest.fixture(scope="module") -def model_repository(): - return TEST_MODEL_REPOSITORY - - -@pytest.fixture(scope="module") -def model(): - return TEST_MODEL - - -@pytest.fixture(scope="module") -def backend(): - return TEST_BACKEND - - -@pytest.fixture(scope="module") -def tokenizer_model(): - return TEST_TOKENIZER - - -@pytest.fixture(scope="module") -def prompt(): - return TEST_PROMPT - - -@pytest.fixture(scope="module") -def messages(): - return TEST_MESSAGES - - # FIXME: In TRTLLM tests, the in-process Triton server for the FastAPI app # does not automatically release GPU memory, even after calling stop(). # The memory is only released when the entire pytest process exits. diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 5402be451d..565146f7c7 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -35,7 +35,6 @@ from tests.utils import setup_fastapi_app, setup_server -@pytest.mark.fastapi class TestChatCompletions: @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): @@ -564,7 +563,6 @@ def test_usage_response( # For tests that won't use the same pytest fixture for server startup across # the whole class test suite. -@pytest.mark.fastapi class TestChatCompletionsTokenizers: # Re-use a single Triton server for different frontend configurations @pytest.fixture(scope="class") diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 9ec3ffe7f7..31eb2f8ae6 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -29,7 +29,6 @@ import pytest -@pytest.mark.fastapi class TestCompletions: @pytest.fixture(scope="class") def client(self, fastapi_client_class_scope): diff --git a/python/openai/tests/test_embeddings.py b/python/openai/tests/test_embeddings.py new file mode 100644 index 0000000000..b2a21b78e9 --- /dev/null +++ b/python/openai/tests/test_embeddings.py @@ -0,0 +1,609 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import base64 +import os +from pathlib import Path + +import numpy as np +import pytest + +# Results on A6000 GPU. The results vary slightly across GPU models. +EMBEDDING_OUTPUT_FLOAT = [ + -0.1914404183626175, + 0.4000193178653717, + 0.058502197265625, + 0.18909454345703125, + -0.4690297544002533, + 0.004936536308377981, + 0.45893096923828125, + -0.31141534447669983, + 0.18299102783203125, + -0.4907582700252533, + 0.6920369267463684, + -0.001537322998046875, + 0.1219015121459961, + -0.11682561784982681, + 0.02811431884765625, + -0.5207672119140625, + 0.4574941098690033, + -0.31097412109375, + 0.13371849060058594, + -0.4693959653377533, + -0.2766602337360382, + 0.029005050659179688, + -0.13730454444885254, + -0.18662643432617188, + 0.0063533782958984375, + 0.16905848681926727, + 0.1612701416015625, + 0.08376502990722656, + -0.09822845458984375, + -0.012738545425236225, + -0.16643650829792023, + -0.01901753805577755, + -0.0503285713493824, + 0.03994830325245857, + -0.3373819887638092, + -0.0188166294246912, + 0.374481201171875, + -0.4371846616268158, + 0.22470474243164062, + -0.011973063461482525, + 0.13784568011760712, + -0.1484222412109375, + 0.19347667694091797, + 0.11848513036966324, + 0.09258091449737549, + -0.2887814939022064, + -0.2301533967256546, + -0.12088584899902344, + 0.1941477507352829, + -0.05228869244456291, + -0.2508443295955658, + 0.15698719024658203, + -0.19024403393268585, + -0.4728952944278717, + 0.336700439453125, + 0.11721674352884293, + 0.24498240649700165, + -0.4826914370059967, + -0.119984470307827, + 0.1008249893784523, + -0.0983428955078125, + -0.0059178671799600124, + 0.2341969758272171, + 0.2725118100643158, + 0.2384185791015625, + -0.30748113989830017, + -0.17387008666992188, + -0.44342041015625, + -0.15537135303020477, + 0.27348455786705017, + 0.1131540909409523, + -0.23855717480182648, + 0.2574056088924408, + -0.12090619653463364, + -0.14412720501422882, + -0.2408244013786316, + 0.0021938085556030273, + -0.39945730566978455, + -0.555908203125, + 0.0760548934340477, + -0.1530914306640625, + -0.40975189208984375, + -0.2091045379638672, + 0.20317332446575165, + -0.20295588672161102, + -0.3643442690372467, + 0.05287488177418709, + -0.24874623119831085, + 0.11500009149312973, + 0.1661122590303421, + 0.26618704199790955, + -0.22980372607707977, + -0.202911376953125, + -0.2738393247127533, + 0.20629756152629852, + -0.24571101367473602, + -0.1486002653837204, + -0.12444128841161728, + 0.27539315819740295, + 0.41679826378822327, + -0.01199467945843935, + 0.1778361052274704, + -0.15123574435710907, + -0.0391184501349926, + -0.035979270935058594, + 0.11838880926370621, + -0.07832065969705582, + 0.15302227437496185, + -0.11540285497903824, + -0.008619308471679688, + 0.011735956184566021, + 0.41825103759765625, + 0.1798756867647171, + 0.0468953438103199, + -0.31410470604896545, + -0.28439536690711975, + 0.028476715087890625, + -0.18972015380859375, + -0.1492512971162796, + 0.23354721069335938, + 0.2631734311580658, + 0.3009694516658783, + -0.31204381585121155, + 0.17155838012695312, + -0.6126009821891785, + -0.16471035778522491, + 0.7154337763786316, + 0.0, + -0.3936564028263092, + -0.15255196392536163, + 0.24118296802043915, + -0.13930638134479523, + 0.6811599731445312, + 0.135009765625, + -0.18750762939453125, + 0.26521047949790955, + -0.1257190704345703, + 0.0532684326171875, + 0.25982680916786194, + -0.3410797119140625, + -0.189666748046875, + 0.016697248443961143, + 0.1474812775850296, + 0.085713230073452, + -0.0862935408949852, + 0.521209716796875, + 0.3840688169002533, + 0.04953320696949959, + -0.0478159599006176, + -0.3888498842716217, + 0.3243462145328522, + 0.03093973733484745, + -0.3594563901424408, + 0.16615693271160126, + -0.07209650427103043, + 0.049218177795410156, + 0.14628247916698456, + -0.10561561584472656, + 0.1696879118680954, + 0.1195220947265625, + 0.0140139264985919, + 0.08987680822610855, + 0.02198282815515995, + -0.06835142523050308, + -0.09100532531738281, + -0.3970082700252533, + -0.20552189648151398, + -0.0871327742934227, + -0.008806228637695312, + 0.10437265783548355, + 0.2754974365234375, + 0.2630208432674408, + -0.67779541015625, + 0.32654380798339844, + -0.4008077085018158, + 0.2785542905330658, + 0.16632080078125, + -0.0709940567612648, + -0.1678619384765625, + -0.11333879083395004, + 0.5577189326286316, + 0.3165779113769531, + -0.2243397980928421, + 0.08053144067525864, + 0.1904652863740921, + 0.22478973865509033, + 0.11852264404296875, + -0.2071024626493454, + 0.2380015105009079, + 0.4622955322265625, + 0.1029459610581398, + -0.30094656348228455, + 0.0351104736328125, + -0.09827486425638199, + 0.0018183389911428094, + 0.07406362146139145, + 0.18090057373046875, + 0.2231648713350296, + -0.1001536026597023, + 0.06609535217285156, + 0.0055376687087118626, + -0.02939859963953495, + -0.17679977416992188, + 0.2300567626953125, + -0.232757568359375, + -0.1863892823457718, + 0.14040501415729523, + -0.21081669628620148, + 0.4772237241268158, + 0.00708770751953125, + 0.25393548607826233, + -0.12926609814167023, + -0.21408335864543915, + 0.43414306640625, + -0.16021983325481415, + -0.6590754389762878, + 0.383026123046875, + 0.4894002377986908, + -0.5350291132926941, + 0.1563262939453125, + 0.4013887941837311, + -0.1429697722196579, + -0.1266673356294632, + 0.0, + -0.12781651318073273, + 0.5082905888557434, + -0.4895477294921875, + 0.05857785418629646, + -0.01038360595703125, + -0.4025942385196686, + -0.6376139521598816, + -0.27256616950035095, + -0.2183430939912796, + 0.13019943237304688, + -0.2378387451171875, + -0.12579791247844696, + 0.23233287036418915, + -0.1948690414428711, + -0.10780048370361328, + 0.4768002927303314, + 0.2761942446231842, + 0.09968694299459457, + -0.07807016372680664, + 0.18632762134075165, + -0.014780680648982525, + 0.18301646411418915, + 0.10943603515625, + 0.45223236083984375, + -0.24634425342082977, + 0.5127970576286316, + 0.15272267162799835, + 0.26901498436927795, + -0.8670451045036316, + -0.20471616089344025, + 0.3934173583984375, + -0.22558848559856415, + 0.14676158130168915, + -0.16282017529010773, + 0.0047810873948037624, + 0.49467912316322327, + -0.1040293350815773, + -0.13565094769001007, + -0.05704273656010628, + 0.2030487060546875, + 0.27226924896240234, + -0.16900062561035156, + 0.06879997253417969, + 0.44347524642944336, + 0.08619359880685806, + -0.1269734650850296, + -0.05267079547047615, + -0.3465728759765625, + 0.1846415251493454, + -0.0655873641371727, + 0.027518590912222862, + -0.06689834594726562, + -0.13316090404987335, + -0.3649355471134186, + -0.0573628731071949, + 0.030780792236328125, + 0.2462870329618454, + -0.0250523891299963, + 0.08964482694864273, + -0.34076571464538574, + -0.3342704772949219, + -0.000331878662109375, + 0.25020280480384827, + 0.34731578826904297, + 0.4081510007381439, + 0.0661773681640625, + 0.14612038433551788, + -0.37111154198646545, + -0.17901070415973663, + 0.0565798282623291, + -0.1689503937959671, + 0.311676025390625, + 0.06296539306640625, + 0.11648496240377426, + -0.16365115344524384, + -0.011795361526310444, + -0.4601001739501953, + 0.13840866088867188, + 0.1115519180893898, + -0.3645426332950592, + -0.182403564453125, + -0.20782725512981415, + -0.004481792449951172, + 0.0870104655623436, + -0.11704126745462418, + 0.34148290753364563, + 0.17841561138629913, + -0.2754109799861908, + -0.0867462158203125, + 0.09910837560892105, + -0.14540545642375946, + -0.10996246337890625, + -0.10946687310934067, + 0.023001352325081825, + 0.11987527459859848, + -5.960464477539063e-8, + 0.3316993713378906, + -0.025622526183724403, + -0.28015899658203125, + 0.34741735458374023, + 0.04091135784983635, + -0.34874120354652405, + 0.22758229076862335, + -0.042999267578125, + 0.0382130928337574, + 0.5654922127723694, + -0.9378255009651184, + 0.17114512622356415, + 0.13035202026367188, + 0.4369252622127533, + 0.0897369384765625, + 0.19928233325481415, + 0.33091607689857483, + -0.10624822229146957, + -0.2845611572265625, + 0.2822163999080658, + 0.1722426414489746, + 0.2111460417509079, + -0.1069692000746727, + -0.3496347963809967, + 0.15000660717487335, + 0.014147520065307617, + -0.36633554100990295, + 0.23989041149616241, + -0.06397350877523422, + 0.2501627504825592, + 0.04016287997364998, + -0.3789469301700592, + -0.4247843325138092, + 0.1515035629272461, + 0.36554718017578125, + 0.057392120361328125, + -0.3492482602596283, + -0.45532989501953125, + 0.4090474545955658, + -0.3914286196231842, + -0.4888407289981842, + 0.4746551513671875, + -0.6188761591911316, + -0.018857955932617188, + 0.02373504638671875, + 0.22691090404987335, + -0.07608286291360855, + 0.5331514477729797, + -0.27182260155677795, + 0.2309315949678421, + -0.1824493408203125, + 0.12648265063762665, + 0.2586142122745514, + -0.07648912817239761, + 0.2318166047334671, + -0.5225245356559753, + 0.133880615234375, + -0.010974247939884663, + 0.09001413732767105, + 0.2562611997127533, + 0.19260406494140625, + 0.4470011293888092, + -0.1636505126953125, + -0.3675130307674408, +] + + +@pytest.mark.skipif( + os.environ.get("IMAGE_KIND") == "TRTLLM", + reason="TRT-LLM backend does not support embedding requests", +) +class TestEmbeddings: + @pytest.fixture(scope="class") + def client(self, fastapi_client_class_scope): + yield fastapi_client_class_scope + + @pytest.fixture(scope="class") + def model(self): + # Override with embeddings-specific model + return "all-MiniLM-L6-v2" + + @pytest.fixture(scope="class") + def tokenizer_model(self): + return None + + @pytest.fixture(scope="class") + def model_repository(self): + # Override with embeddings-specific repository + return str(Path(__file__).parent / "vllm_embedding_models") + + @pytest.fixture(scope="class") + def input(self): + return "The food was delicious and the waiter..." + + def _check_embedding_response( + self, response, model, dims=len(EMBEDDING_OUTPUT_FLOAT), encoding_format="float" + ): + assert response.status_code == 200, response.json() + embedding = response.json()["data"][0]["embedding"] + assert embedding is not None + if encoding_format == "base64": + embedding = np.frombuffer(base64.b64decode(embedding), dtype=np.float32) + + # The results vary slightly across GPU models + result = np.allclose( + EMBEDDING_OUTPUT_FLOAT[:dims], embedding, rtol=0, atol=1e-3 + ) + assert ( + result + ), f"Embeddings do not match expected output\nExpect {EMBEDDING_OUTPUT_FLOAT[:dims]},\ngot{embedding}" + + assert response.json()["data"][0]["object"] == "embedding" + assert response.json()["data"][0]["index"] == 0 + assert response.json()["model"] == model + + usage = response.json().get("usage") + assert usage is not None + assert usage["prompt_tokens"] == 12 + assert usage["total_tokens"] == 12 + + @pytest.mark.parametrize( + "input", + [ + "The food was delicious and the waiter...", + [101, 1996, 2833, 2001, 12090, 1998, 1996, 15610, 1012, 1012, 1012, 102], + ], + ) + def test_embeddings_defaults(self, client, model: str, input: str): + response = client.post( + "/v1/embeddings", + json={"model": model, "input": input}, + ) + + self._check_embedding_response(response, model) + + # FIXME: Python model cannot unload gracefully if raise error. + # def test_chat_completions_defaults( + # self, client, model: str, messages: List[dict], backend: str + # ): + # response = client.post( + # "/v1/chat/completions", + # json={"model": model, "messages": messages}, + # ) + + # assert response.status_code == 400 + # assert "does not support" in response.json()["detail"] + + @pytest.mark.parametrize( + "param_key, param_value", + [ + ("dimensions", [10]), + ("encoding_format", "invalid"), + ("encoding_format", 0), + ], + ) + def test_embeddings_invalid_parameters( + self, client, param_key, param_value, model: str, input: str + ): + response = client.post( + "/v1/embeddings", + json={ + "model": model, + "input": input, + param_key: param_value, + }, + ) + + # Assert schema validation error + assert response.status_code == 422, response.json() + + @pytest.mark.parametrize("dimensions", [0, 10, 100, -1]) + @pytest.mark.parametrize("encoding_format", ["float", "base64"]) + def test_embeddings_parameters( + self, client, dimensions, encoding_format, model: str, input: str + ): + response = client.post( + "/v1/embeddings", + json={ + "model": model, + "input": input, + "dimensions": dimensions, + "encoding_format": encoding_format, + }, + ) + + self._check_embedding_response( + response, model, dims=dimensions, encoding_format=encoding_format + ) + + def test_embeddings_empty_request(self, client): + response = client.post("/v1/embeddings", json={}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Field required" + + def test_embeddings_no_model(self, client, input: str): + response = client.post("/v1/embeddings", json={"input": input}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Field required" + + @pytest.mark.parametrize( + "model, error_code", + [ + ("", 400), + (123, 422), + ("Invalid", 400), + (None, 422), + ], + ) + def test_embeddings_invalid_model(self, client, model: str, input, error_code: int): + print("Model:", model) + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == error_code + if error_code == 400: + assert response.json()["detail"] == f"Unknown model: {model}" + else: + assert ( + response.json()["detail"][0]["msg"] == "Input should be a valid string" + ) + + def test_embeddings_no_input(self, client, model: str): + response = client.post("/v1/embeddings", json={"model": model}) + assert response.status_code == 422 + + @pytest.mark.parametrize( + "input", + [ + "", + [], + ], + ) + def test_embeddings_empty_input(self, client, model: str, input): + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == 422 + assert ( + response.json()["detail"][0]["msg"] + == "Value should have at least 1 item after validation, not 0" + ) + + @pytest.mark.parametrize( + "input", + [ + 123, + 1.5, + 0, + None, + ], + ) + def test_embeddings_invalid_input(self, client, model: str, input): + # Message validation requires min_length of 1 + response = client.post("/v1/embeddings", json={"model": model, "input": input}) + assert response.status_code == 422 + assert response.json()["detail"][0]["msg"] == "Input should be a valid string" diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json new file mode 100644 index 0000000000..2ad058c2e5 --- /dev/null +++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/1/model.json @@ -0,0 +1 @@ +{"model": "sentence-transformers/all-MiniLM-L6-v2", "gpu_memory_utilization": 0.5} diff --git a/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt new file mode 100644 index 0000000000..39b3c48edb --- /dev/null +++ b/python/openai/tests/vllm_embedding_models/all-MiniLM-L6-v2/config.pbtxt @@ -0,0 +1,28 @@ +# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +backend: "vllm" +instance_group [{kind: KIND_MODEL}]