From 573759309662a9ce6373a2e54e013af88f8b8084 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 23:15:48 +0800 Subject: [PATCH 1/3] FEAT: report cache hit tokens across vLLM, SGLang and MLX --- .../de/LC_MESSAGES/user_guide/client_api.po | 12 + .../es/LC_MESSAGES/user_guide/client_api.po | 12 + .../fr/LC_MESSAGES/user_guide/client_api.po | 12 + .../it/LC_MESSAGES/user_guide/client_api.po | 12 + .../ja/LC_MESSAGES/user_guide/client_api.po | 12 + .../ko/LC_MESSAGES/user_guide/client_api.po | 12 + .../LC_MESSAGES/user_guide/client_api.po | 12 + .../LC_MESSAGES/user_guide/client_api.po | 12 + .../LC_MESSAGES/user_guide/client_api.po | 12 + doc/source/user_guide/client_api.rst | 7 + xinference/model/llm/mlx/core.py | 13 + xinference/model/llm/mlx/tests/test_xavier.py | 40 ++- xinference/model/llm/sglang/core.py | 43 +-- .../llm/tests/test_cached_token_usage.py | 246 ++++++++++++++++++ xinference/model/llm/vllm/core.py | 54 ++-- 15 files changed, 455 insertions(+), 56 deletions(-) create mode 100644 xinference/model/llm/tests/test_cached_token_usage.py diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po index 66f3bafb81..66e29eae42 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po @@ -209,3 +209,15 @@ msgstr "Starten Sie das Rerank-Modell und berechnen Sie die Textähnlichkeit:" #~ "Bei der Verwendung von Anthropic zum Senden von Anfragen bleibt die " #~ "Kompatibilität mit der Anthropic-Schnittstelle erhalten, mit Ausnahme der " #~ "Modellerstellung." + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "Tokenanzahl bei Cache-Treffern" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "Bei vLLM, SGLang und MLX geben Completion- und Chat-Completion-Antworten die von der Engine gemeldete Anzahl wiederverwendeter Prompt-Token in ``usage.prompt_tokens_details.cached_tokens`` zurück, sofern verfügbar. Diese Anzahl umfasst die von der Engine gemeldete Wiederverwendung aus dem nativen Präfix-Cache und dem Xavier-Cache. Sie ist Teil von ``prompt_tokens`` und keine zusätzliche Tokenanzahl." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "Setzen Sie bei Streaming-Anfragen ``stream_options={\"include_usage\": True}``, um den abschließenden Nutzungs-Chunk mit ``choices=[]`` zu erhalten. Die Responses API gibt dieselbe Anzahl als ``usage.input_tokens_details.cached_tokens`` zurück. Wenn die Engine keine Cache-Treffer meldet, lassen Completion- und Chat-Completion-Antworten ``prompt_tokens_details`` weg; die Responses API verwendet standardmäßig null." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po index 7da6fc5d77..ef239dd717 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po @@ -207,3 +207,15 @@ msgstr "Inicia el modelo rerank y calcula la similitud de texto:" #~ "Anthropic client request with the same function as before, excluding launch " #~ "model." #~ msgstr "" + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "Número de tokens encontrados en caché" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "Con vLLM, SGLang y MLX, las respuestas de completado y completado de chat exponen el número de tokens del prompt reutilizados por el motor en ``usage.prompt_tokens_details.cached_tokens``, cuando está disponible. Este número incluye la reutilización de la caché de prefijos nativa y de la caché Xavier que informa el motor; forma parte de ``prompt_tokens`` y no es un número adicional de tokens." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "Para las solicitudes en streaming, establezca ``stream_options={\"include_usage\": True}`` para recibir el bloque final de uso con ``choices=[]``. La Responses API expone el mismo número como ``usage.input_tokens_details.cached_tokens``. Si el motor no informa de aciertos de caché, las respuestas de completado y completado de chat omiten ``prompt_tokens_details``; la Responses API utiliza cero de forma predeterminada." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po index f15ba5ce70..9c2a7978be 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po @@ -211,3 +211,15 @@ msgstr "Lancez le modèle rerank et calculez la similarité textuelle :" #~ "Lors de l'envoi de requêtes avec Anthropic, toutes les requêtes, à " #~ "l'exception de la création de modèles, restent compatibles avec l'interface " #~ "d'Anthropic." + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "Nombre de tokens trouvés en cache" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "Avec vLLM, SGLang et MLX, les réponses de complétion et de complétion de chat exposent le nombre de tokens du prompt réutilisés par le moteur dans ``usage.prompt_tokens_details.cached_tokens``, lorsque cette information est disponible. Ce nombre inclut la réutilisation du cache de préfixes natif et du cache Xavier signalée par le moteur ; il fait partie de ``prompt_tokens`` et ne constitue pas un nombre de tokens supplémentaire." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "Pour les requêtes en streaming, définissez ``stream_options={\"include_usage\": True}`` afin de recevoir le dernier bloc de statistiques avec ``choices=[]``. La Responses API expose le même nombre dans ``usage.input_tokens_details.cached_tokens``. Si le moteur ne signale pas les succès de cache, les réponses de complétion et de complétion de chat omettent ``prompt_tokens_details`` ; la Responses API utilise zéro par défaut." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po index 7b356a9952..b60662c29f 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po @@ -210,3 +210,15 @@ msgstr "Caricare il modello di rerank e calcolare la similarità testuale:" #~ "Quando si inviano richieste utilizzando Anthropic, ad eccezione della " #~ "creazione del modello, tutte le altre richieste rimangono compatibili con " #~ "l'interfaccia di Anthropic." + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "Numero di token trovati nella cache" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "Con vLLM, SGLang e MLX, le risposte di completamento e completamento chat espongono il numero di token del prompt riutilizzati dal motore in ``usage.prompt_tokens_details.cached_tokens``, quando disponibile. Questo numero include il riutilizzo della cache dei prefissi nativa e della cache Xavier riportato dal motore; fa parte di ``prompt_tokens`` e non rappresenta un numero aggiuntivo di token." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "Per le richieste in streaming, impostare ``stream_options={\"include_usage\": True}`` per ricevere il blocco finale delle statistiche di utilizzo con ``choices=[]``. La Responses API espone lo stesso numero come ``usage.input_tokens_details.cached_tokens``. Se il motore non riporta gli accessi riusciti alla cache, le risposte di completamento e completamento chat omettono ``prompt_tokens_details``; la Responses API utilizza zero come valore predefinito." diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po index fdefa16504..8af276d337 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po @@ -192,3 +192,15 @@ msgstr "rerankモデルを起動し、テキスト類似度を計算する：" #~ msgstr "" #~ "Anthropic を使用してリクエストを送信する場合、モデルの作成を除き、その他のリクエストはすべて Anthropic " #~ "のインターフェースとの互換性を維持します。" + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "キャッシュヒットしたトークン数" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "vLLM、SGLang、MLX では、エンジンが統計を提供する場合、補完およびチャット補完のレスポンスの ``usage.prompt_tokens_details.cached_tokens`` に、再利用されたプロンプトトークン数が含まれます。この数値には、エンジンが報告したネイティブのプレフィックスキャッシュと Xavier キャッシュの再利用が含まれます。これは ``prompt_tokens`` の一部であり、追加のトークン数ではありません。" + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "ストリーミングリクエストでは、``stream_options={\"include_usage\": True}`` を設定すると、``choices=[]`` の最終使用量チャンクを受け取れます。Responses API は同じ数値を ``usage.input_tokens_details.cached_tokens`` で返します。エンジンがキャッシュヒットを報告しない場合、補完およびチャット補完のレスポンスでは ``prompt_tokens_details`` が省略され、Responses API では既定値のゼロが返されます。" diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po index ab59ffa7e7..47b9b3b522 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po @@ -193,3 +193,15 @@ msgstr "rerank 모델을 불러와 텍스트 유사도를 계산합니다:" #~ "model." #~ msgstr "" #~ "Anthropic을 사용하여 요청을 보낼 때, 모델 생성 외의 모든 요청은 Anthropic의 인터페이스와 호환성을 유지합니다." + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "캐시 적중 토큰 수" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "vLLM, SGLang, MLX에서 엔진이 통계를 제공하면 완성 및 채팅 완성 응답의 ``usage.prompt_tokens_details.cached_tokens``에 재사용된 프롬프트 토큰 수가 포함됩니다. 이 값에는 엔진이 보고한 기본 접두사 캐시 재사용과 Xavier 캐시 재사용이 포함됩니다. ``prompt_tokens``의 일부이며 추가 토큰 수가 아닙니다." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "스트리밍 요청에서 ``stream_options={\"include_usage\": True}``를 설정하면 ``choices=[]``인 최종 사용량 청크를 받을 수 있습니다. Responses API는 동일한 값을 ``usage.input_tokens_details.cached_tokens``로 반환합니다. 엔진이 캐시 적중을 보고하지 않으면 완성 및 채팅 완성 응답에서 ``prompt_tokens_details``가 생략되고 Responses API는 기본값인 0을 반환합니다." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po index bf2245adde..2ea767c523 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po @@ -207,3 +207,15 @@ msgstr "Iniciar o modelo de rerank e calcular a similaridade textual:" #~ msgstr "" #~ "Ao enviar requisições usando Anthropic, exceto para criar modelos, todas as " #~ "outras requisições mantêm compatibilidade com a interface do Anthropic." + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "Quantidade de tokens encontrados no cache" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "Com vLLM, SGLang e MLX, as respostas de completamento e completamento de chat expõem a quantidade de tokens do prompt reutilizados pelo mecanismo em ``usage.prompt_tokens_details.cached_tokens``, quando disponível. Essa quantidade inclui a reutilização do cache de prefixos nativo e do cache Xavier informada pelo mecanismo; faz parte de ``prompt_tokens`` e não é uma quantidade adicional de tokens." + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "Para solicitações em streaming, defina ``stream_options={\"include_usage\": True}`` para receber o bloco final de uso com ``choices=[]``. A Responses API expõe a mesma quantidade como ``usage.input_tokens_details.cached_tokens``. Se o mecanismo não informar acertos de cache, as respostas de completamento e completamento de chat omitem ``prompt_tokens_details``; a Responses API usa zero por padrão." diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po index 298ba01ce6..42e51c8012 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po @@ -227,3 +227,15 @@ msgstr "响应包含单个向量，与文本 Embeddings API 返回的列表形 #: ../../source/user_guide/client_api.rst:430 msgid "Compare vectors with cosine similarity for speaker verification or speaker identification. See :ref:`audio` for Web UI usage, input processing details, and the complete Speaker Embedding API example." msgstr "可以使用余弦相似度比较向量，进行说话人验证或识别。Web UI 使用方法、输入处理细节和完整的说话人嵌入 API 示例参见 :ref:`audio`。" + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "缓存命中 token 数" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "使用 vLLM、SGLang 和 MLX 时，如果引擎提供统计，补全和聊天补全响应会通过 ``usage.prompt_tokens_details.cached_tokens`` 返回复用的提示 token 数。该数值包含引擎报告的原生前缀缓存复用和 Xavier 缓存复用，是 ``prompt_tokens`` 的一部分，不是额外的 token 数。" + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "流式请求设置 ``stream_options={\"include_usage\": True}`` 后，会收到 ``choices=[]`` 的最终用量统计块。Responses API 通过 ``usage.input_tokens_details.cached_tokens`` 返回同一数值。如果引擎未报告缓存命中，补全和聊天补全响应会省略 ``prompt_tokens_details``；Responses API 则默认返回零。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po index efb2cfd35c..cf1100d04d 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po @@ -188,3 +188,15 @@ msgstr "拉起 rerank 模型並計算文字相似度：" #~ "Anthropic client request with the same function as before, excluding launch " #~ "model." #~ msgstr "使用 Anthropic 發送請求時，除了創建模型，其餘請求皆保持與 Anthropic 的接口相容。" + +#: user_guide/client_api.rst:96 +msgid "Cache hit token counts" +msgstr "快取命中 token 數" + +#: user_guide/client_api.rst:99 +msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." +msgstr "使用 vLLM、SGLang 和 MLX 時，如果引擎提供統計，補全和聊天補全回應會透過 ``usage.prompt_tokens_details.cached_tokens`` 傳回重用的提示 token 數。此數值包含引擎回報的原生前綴快取重用和 Xavier 快取重用，是 ``prompt_tokens`` 的一部分，不是額外的 token 數。" + +#: user_guide/client_api.rst:101 +msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." +msgstr "串流請求設定 ``stream_options={\"include_usage\": True}`` 後，會收到 ``choices=[]`` 的最終用量統計區塊。Responses API 透過 ``usage.input_tokens_details.cached_tokens`` 傳回相同數值。如果引擎未回報快取命中，補全和聊天補全回應會省略 ``prompt_tokens_details``；Responses API 則預設傳回零。" diff --git a/doc/source/user_guide/client_api.rst b/doc/source/user_guide/client_api.rst index e439b67699..9b2ab364d4 100644 --- a/doc/source/user_guide/client_api.rst +++ b/doc/source/user_guide/client_api.rst @@ -93,6 +93,13 @@ More details refer to: https://platform.openai.com/docs/api-reference/chat?lang= max_tokens=1024 ) +Cache hit token counts +====================== + +With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count. + +For streaming requests, set ``stream_options={"include_usage": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero. + OpenAI Client Tool Calls ======================== diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index 63a1fb9d79..e39e41c2fa 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -1511,6 +1511,7 @@ async def async_generate( if stream: # Return async generator for streaming async def stream_generator(): + last_chunk = None async for chunk in self._batch_model.generate_stream( prompt=prompt_text, max_tokens=max_tokens, @@ -1531,8 +1532,20 @@ async def stream_generator(): chunk["model"] = self.model_uid if chunk.get("choices") and len(chunk["choices"]) > 0: chunk["choices"][0]["index"] = 0 + last_chunk = chunk yield chunk + stream_options = generate_config.get("stream_options") + if ( + isinstance(stream_options, dict) + and stream_options.get("include_usage") + and last_chunk is not None + and last_chunk.get("usage") is not None + ): + usage_chunk = last_chunk.copy() + usage_chunk["choices"] = [] + yield usage_chunk + return stream_generator() else: # Non-streaming: get full result and return completion diff --git a/xinference/model/llm/mlx/tests/test_xavier.py b/xinference/model/llm/mlx/tests/test_xavier.py index 48eee6e435..65e3ab956c 100644 --- a/xinference/model/llm/mlx/tests/test_xavier.py +++ b/xinference/model/llm/mlx/tests/test_xavier.py @@ -853,8 +853,9 @@ def fail(*args, **kwargs): @metal @pytest.mark.asyncio -async def test_real_batch_generation_reuses_remote_cache(monkeypatch): - from ..core import MLXBatchModel +@pytest.mark.parametrize("stream", [False, True]) +async def test_real_batch_generation_reuses_remote_cache(monkeypatch, stream): + from ..core import MLXBatchModel, MLXModel class Tokenizer: eos_token_ids = [] @@ -879,19 +880,38 @@ async def generate(role=None, transfer=None): client = MLXXavierCache(contract(), dict(cfg, role=role)) client.publish = MagicMock(wraps=client.publish) clients.append(client) + native_model = model() wrapper = MLXBatchModel( - model(), Tokenizer(), prompt_cache_size=0, xavier=client + native_model, Tokenizer(), prompt_cache_size=0, xavier=client ) + adapter = object.__new__(MLXModel) + adapter.model_uid = "mlx-cache-test" + adapter._model = native_model + adapter._tokenizer = Tokenizer() + adapter._model_generation_config = {} + adapter._batch_model = wrapper + adapter._xavier = client try: - tokens = Tokenizer().encode("a" * 70) - prepared = await client.fetch(tokens, transfer) - return await wrapper.generate( + result = await adapter.async_generate( "a" * 70, - 8, - temperature=0, - prepared_cache=prepared, - prompt_token_ids=tokens, + dict( + max_tokens=8, + temperature=0, + stream=stream, + stream_options={"include_usage": True}, + **({"_pd_kv_transfer_params": transfer} if transfer else {}), + ), ) + if stream: + chunks = [chunk async for chunk in result] + assert chunks[-1]["choices"] == [] + text = "".join( + choice["text"] + for chunk in chunks + for choice in chunk["choices"] + ) + return text, chunks[-1]["usage"] + return result["choices"][0]["text"], result["usage"] finally: tasks = [ g["task"] for g in wrapper._batch_generators.values() if g["task"] diff --git a/xinference/model/llm/sglang/core.py b/xinference/model/llm/sglang/core.py index 551f106ed5..c078998265 100644 --- a/xinference/model/llm/sglang/core.py +++ b/xinference/model/llm/sglang/core.py @@ -631,6 +631,18 @@ def match_json( return False, "sglang library is not installed" return True + @staticmethod + def _get_completion_usage(meta_info: Dict) -> CompletionUsage: + usage = CompletionUsage( + prompt_tokens=meta_info["prompt_tokens"], + completion_tokens=meta_info["completion_tokens"], + total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"], + ) + cached_tokens = meta_info.get("cached_tokens") + if cached_tokens is not None: + usage["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return usage + @staticmethod def _convert_state_to_completion_chunk( request_id: str, @@ -657,11 +669,7 @@ def _convert_state_to_completion_chunk( finish_reason=finish_reason, ) ] - usage = CompletionUsage( - prompt_tokens=meta_info["prompt_tokens"], - completion_tokens=meta_info["completion_tokens"], - total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"], - ) + usage = SGLANGModel._get_completion_usage(meta_info) chunk = CompletionChunk( id=request_id, object="text_completion", @@ -695,11 +703,7 @@ def _convert_state_to_completion( ) ] - usage = CompletionUsage( - prompt_tokens=meta_info["prompt_tokens"], - completion_tokens=meta_info["completion_tokens"], - total_tokens=meta_info["prompt_tokens"] + meta_info["completion_tokens"], - ) + usage = SGLANGModel._get_completion_usage(meta_info) return Completion( id=request_id, object="text_completion", @@ -1108,6 +1112,9 @@ async def prepare_handoff(): async def stream_results() -> AsyncGenerator[CompletionChunk, None]: prompt_tokens, completion_tokens, total_tokens = 0, 0, 0 + usage = CompletionUsage( + prompt_tokens=0, completion_tokens=0, total_tokens=0 + ) complete_response = "" match_tool_call_tmp_results: List[CompletionChunk] = [] is_match_tool_call = False @@ -1138,11 +1145,7 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: prompt_tokens = meta_info["prompt_tokens"] completion_tokens = meta_info["completion_tokens"] total_tokens = prompt_tokens + completion_tokens - chunk["usage"] = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + usage = chunk["usage"] if tools: """ The qwen2 tool call returns format like this: @@ -1199,7 +1202,7 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: ) else finish_reason ) - yield generate_completion_chunk( + final_chunk = generate_completion_chunk( "", finish_reason=finish_reason, chunk_id=request_id, @@ -1208,6 +1211,8 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: completion_tokens=completion_tokens, total_tokens=total_tokens, ) + final_chunk["usage"] = usage + yield final_chunk if include_usage: chunk = CompletionChunk( @@ -1217,11 +1222,7 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: model=self.model_uid, choices=[], ) - chunk["usage"] = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + chunk["usage"] = usage yield chunk async def pd_stream(): diff --git a/xinference/model/llm/tests/test_cached_token_usage.py b/xinference/model/llm/tests/test_cached_token_usage.py new file mode 100644 index 0000000000..32a9d9c00d --- /dev/null +++ b/xinference/model/llm/tests/test_cached_token_usage.py @@ -0,0 +1,246 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys +from types import ModuleType, SimpleNamespace + +import pytest +from packaging.version import Version + +from xinference.api.protocols.openai_responses import ( + chat_to_response, + parse_responses_request, + responses_stream_events, +) + +from ..utils import ChatModelMixin, generate_completion_chunk + +MISSING = object() + + +def _usage(cached_tokens): + usage = {"prompt_tokens": 100, "completion_tokens": 2, "total_tokens": 102} + if cached_tokens is not MISSING and cached_tokens is not None: + usage["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return usage + + +@pytest.fixture(params=["vllm", "sglang", "mlx"]) +def model_factory(request, monkeypatch): + def create(cached_tokens): + if request.param == "vllm": + from ..vllm import core + + class SamplingParams: + def __init__(self, n=1, max_tokens=2): + self.max_tokens = max_tokens + + sampling_params = ModuleType("vllm.sampling_params") + sampling_params.SamplingParams = SamplingParams + monkeypatch.setitem(sys.modules, "vllm.sampling_params", sampling_params) + monkeypatch.setattr(core, "VLLM_INSTALLED", False) + monkeypatch.setattr(core, "VLLM_VERSION", Version("0.5.0")) + model = object.__new__(core.VLLMModel) + model._nixl_config = model._xavier_config = None + model.reasoning_parser = None + + async def generate(*args, **kwargs): + for text, token_ids, finished in [ + ("hello", [1], False), + ("hello world", [1, 2], True), + ]: + output = SimpleNamespace( + prompt_token_ids=list(range(100)), + outputs=[ + SimpleNamespace( + text=text, + token_ids=token_ids, + index=0, + logprobs=None, + finish_reason="stop" if finished else None, + ) + ], + finished=finished, + ) + if cached_tokens is not MISSING: + output.num_cached_tokens = cached_tokens + yield output + + model._engine = SimpleNamespace(generate=generate) + # Exercise generation without the worker's process-exit guard. + model.async_generate = core.VLLMModel.async_generate.__wrapped__.__get__( + model + ) + elif request.param == "sglang": + from ..sglang.core import SGLANGModel + + model = object.__new__(SGLANGModel) + + def meta_info(completion_tokens, finished): + meta = dict( + prompt_tokens=100, + completion_tokens=completion_tokens, + finish_reason={"type": "stop"} if finished else None, + ) + if cached_tokens is not MISSING: + meta["cached_tokens"] = cached_tokens + # The total already includes these sources; do not add them again. + meta["cached_tokens_details"] = dict(device=16, host=16, storage=32) + return meta + + async def generate(*args, **kwargs): + return dict(text="hello world", meta_info=meta_info(2, True)) + + async def generate_stream(*args, **kwargs): + yield meta_info(1, False), "hello" + yield meta_info(2, True), " world" + + model._non_stream_generate = generate + model._stream_generate = generate_stream + else: + from ..mlx.core import MLXModel + + model = object.__new__(MLXModel) + model._model = model._tokenizer = object() + + async def generate(*args, **kwargs): + return "hello world", _usage(cached_tokens) + + async def generate_stream(*args, **kwargs): + for text, finished in [("hello world", False), ("", True)]: + chunk = generate_completion_chunk( + text, + finish_reason="stop" if finished else None, + chunk_id="request-1", + model_uid="test-model", + prompt_tokens=100, + completion_tokens=2, + total_tokens=102, + ) + chunk["usage"] = _usage(cached_tokens) + yield chunk + + model._batch_model = SimpleNamespace( + generate=generate, generate_stream=generate_stream + ) + + model.model_uid = "test-model" + model._active_request_ids = set() + model._sanitize_generate_config = lambda config: dict( + {"stream": False, "stream_options": None, "n": 1, "lora_name": None}, + **(config or {}), + ) + return model + + return create + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached_tokens", [0, 64]) +async def test_non_stream_cached_tokens_reach_chat_and_responses( + model_factory, cached_tokens +): + completion = await model_factory(cached_tokens).async_generate( + "prompt", {"max_tokens": 2} + ) + assert completion["usage"] == _usage(cached_tokens) + chat = ChatModelMixin._to_chat_completion(completion) + assert chat["usage"] == _usage(cached_tokens) + response = chat_to_response( + chat, parse_responses_request({"model": "test-model", "input": "prompt"}) + ) + assert response["usage"]["input_tokens_details"] == {"cached_tokens": cached_tokens} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached_tokens", [0, 64]) +@pytest.mark.parametrize("include_usage", [False, True]) +async def test_stream_cached_tokens_reach_final_and_usage_chunks( + model_factory, cached_tokens, include_usage +): + stream = await model_factory(cached_tokens).async_generate( + "prompt", + { + "max_tokens": 2, + "stream": True, + "stream_options": {"include_usage": include_usage}, + }, + ) + chunks = [chunk async for chunk in stream] + final = next( + chunk + for chunk in reversed(chunks) + if chunk["choices"] and chunk["choices"][0]["finish_reason"] + ) + assert final["usage"] == _usage(cached_tokens) + usage_chunks = [chunk for chunk in chunks if not chunk["choices"]] + assert len(usage_chunks) == int(include_usage) + if include_usage: + assert chunks[-1]["choices"] == [] + assert usage_chunks[0]["usage"] == _usage(cached_tokens) + + async def source(): + for chunk in chunks: + yield chunk + + chat_stream = ChatModelMixin._async_to_chat_completion_chunks(source()) + request = parse_responses_request( + {"model": "test-model", "input": "prompt", "stream": True} + ) + events = [ + json.loads(event["data"]) + async for event in responses_stream_events(chat_stream, request) + ] + assert events[-1]["response"]["usage"]["input_tokens_details"] == { + "cached_tokens": cached_tokens + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached_tokens", [MISSING, None]) +@pytest.mark.parametrize("stream", [False, True]) +async def test_unreported_cached_tokens_remain_optional( + model_factory, cached_tokens, stream +): + result = await model_factory(cached_tokens).async_generate( + "prompt", + { + "max_tokens": 2, + "stream": stream, + "stream_options": {"include_usage": True}, + }, + ) + if stream: + chunks = [chunk async for chunk in result] + assert chunks[-1]["choices"] == [] + assert chunks[-1]["usage"] == _usage(cached_tokens) + else: + assert result["usage"] == _usage(cached_tokens) + + +def test_vllm_multiple_outputs_do_not_multiply_cached_tokens(): + from ..vllm.core import VLLMModel + + output = SimpleNamespace( + prompt_token_ids=list(range(100)), + num_cached_tokens=64, + outputs=[SimpleNamespace(token_ids=[1, 2]), SimpleNamespace(token_ids=[3])], + ) + assert VLLMModel._get_completion_usage(output) == dict( + prompt_tokens=100, + completion_tokens=3, + total_tokens=103, + prompt_tokens_details={"cached_tokens": 64}, + ) diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py index 965561eb2d..9512effe21 100644 --- a/xinference/model/llm/vllm/core.py +++ b/xinference/model/llm/vllm/core.py @@ -1734,6 +1734,22 @@ def _convert_request_output_to_completion_chunk( finish_reason, ) + @staticmethod + def _get_completion_usage(request_output: "RequestOutput") -> CompletionUsage: + prompt_tokens = len(request_output.prompt_token_ids) + completion_tokens = sum( + len(output.token_ids) for output in request_output.outputs + ) + usage = CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + cached_tokens = getattr(request_output, "num_cached_tokens", None) + if cached_tokens is not None: + usage["prompt_tokens_details"] = {"cached_tokens": cached_tokens} + return usage + @staticmethod def _convert_request_output_to_completion( request_id: str, model: str, request_output: "RequestOutput" @@ -1751,15 +1767,7 @@ def _convert_request_output_to_completion( ) ) - prompt_tokens = len(request_output.prompt_token_ids) - completion_tokens = sum( - len(output.token_ids) for output in request_output.outputs - ) - usage = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) + usage = VLLMModel._get_completion_usage(request_output) return Completion( id=request_id, object="text_completion", @@ -2112,6 +2120,9 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: previous_texts = [""] * sanitized_generate_config["n"] previous_logprobs_counts = [0] * sanitized_generate_config["n"] prompt_tokens, completion_tokens, total_tokens = 0, 0, 0 + usage = CompletionUsage( + prompt_tokens=0, completion_tokens=0, total_tokens=0 + ) complete_response = "" match_tool_call_tmp_results = [] is_match_tool_call = False @@ -2139,16 +2150,11 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: previous_logprobs_counts[i] = current_count complete_response += delta - prompt_tokens = len(_request_output.prompt_token_ids) - completion_tokens = sum( - len(output.token_ids) for output in _request_output.outputs - ) - total_tokens = prompt_tokens + completion_tokens - chunk["usage"] = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + usage = self._get_completion_usage(_request_output) + prompt_tokens = usage["prompt_tokens"] + completion_tokens = usage["completion_tokens"] + total_tokens = usage["total_tokens"] + chunk["usage"] = usage if tools: """ @@ -2211,7 +2217,7 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: ) # match OpenAI API stream - yield generate_completion_chunk( + final_chunk = generate_completion_chunk( chunk_text="", finish_reason=finish_reason, chunk_id=request_id, @@ -2220,6 +2226,8 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: completion_tokens=completion_tokens, total_tokens=total_tokens, ) + final_chunk["usage"] = usage + yield final_chunk if include_usage: chunk = CompletionChunk( @@ -2229,11 +2237,7 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: model=self.model_uid, choices=[], ) - chunk["usage"] = CompletionUsage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=total_tokens, - ) + chunk["usage"] = usage yield chunk if stream: From 48c5650f5e548f19b40aa2ba693e30d8be9b5996 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 23:47:42 +0800 Subject: [PATCH 2/3] TEST: preserve cached token usage through heterogeneous PD --- xinference/core/tests/test_pd_model.py | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/xinference/core/tests/test_pd_model.py b/xinference/core/tests/test_pd_model.py index 56e2982214..cfd13eda84 100644 --- a/xinference/core/tests/test_pd_model.py +++ b/xinference/core/tests/test_pd_model.py @@ -1,6 +1,7 @@ # Copyright 2022-2026 Xinference Holdings Pte. Ltd # Licensed under the Apache License, Version 2.0. import asyncio +import json from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock @@ -135,6 +136,51 @@ async def d(*args, **kwargs): assert not actor._request_set and not actor._direct_transfers +@pytest.mark.asyncio +@pytest.mark.parametrize("method", ["chat", "generate"]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("mode", ["gpu", "host"]) +async def test_cross_engine_preserves_decode_cached_token_usage( + router, monkeypatch, method, stream, mode +): + actor, prefill, decode = router + actor._model_engine = "heterogeneous" + actor._handoff_mode = mode + monkeypatch.setattr(xo, "actor_ref", AsyncMock(return_value=AsyncMock())) + usage = dict( + prompt_tokens=193, + completion_tokens=2, + total_tokens=195, + prompt_tokens_details={"cached_tokens": 192}, + ) + # P's cache hits must not replace D's imported-token count or be added to it. + getattr(prefill, method).return_value = dict( + choices=[], usage=dict(usage, prompt_tokens_details={"cached_tokens": 64}) + ) + response = json.dumps( + dict(choices=[{"finish_reason": "stop"}], usage=usage) + ).encode() + usage_chunk = json.dumps(dict(choices=[], usage=usage)).encode() + + async def chunks(): + yield response + yield usage_chunk + + getattr(decode, method).return_value = chunks() if stream else response + result = await actor._infer( + method, + "prompt", + {"stream": stream, "stream_options": {"include_usage": True}}, + request_id="r", + ) + if stream: + assert [chunk async for chunk in result] == [response, usage_chunk] + decode.decrease_serve_count.assert_awaited_once() + else: + assert result == response + assert not actor._request_set and not actor._direct_transfers + + @pytest.mark.asyncio @pytest.mark.parametrize("backend", ["xavier", "nixl"]) async def test_sglang_stream_disconnect_aborts_both_roles(router, monkeypatch, backend): From 2738002f50a2766ba292f5ca3352f3363124b81c Mon Sep 17 00:00:00 2001 From: qinxuye Date: Thu, 8 Oct 2026 00:27:40 +0800 Subject: [PATCH 3/3] FIX: clarify PD cache usage and cover tool responses --- .../de/LC_MESSAGES/user_guide/client_api.po | 6 +- .../es/LC_MESSAGES/user_guide/client_api.po | 6 +- .../fr/LC_MESSAGES/user_guide/client_api.po | 6 +- .../it/LC_MESSAGES/user_guide/client_api.po | 6 +- .../ja/LC_MESSAGES/user_guide/client_api.po | 6 +- .../ko/LC_MESSAGES/user_guide/client_api.po | 6 +- .../LC_MESSAGES/user_guide/client_api.po | 6 +- .../LC_MESSAGES/user_guide/client_api.po | 6 +- .../LC_MESSAGES/user_guide/client_api.po | 6 +- doc/source/user_guide/client_api.rst | 2 + .../llm/tests/test_cached_token_usage.py | 100 +++++++++++++++++- 11 files changed, 144 insertions(+), 12 deletions(-) diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po index 66e29eae42..50020e4ce0 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/client_api.po @@ -218,6 +218,10 @@ msgstr "Tokenanzahl bei Cache-Treffern" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "Bei vLLM, SGLang und MLX geben Completion- und Chat-Completion-Antworten die von der Engine gemeldete Anzahl wiederverwendeter Prompt-Token in ``usage.prompt_tokens_details.cached_tokens`` zurück, sofern verfügbar. Diese Anzahl umfasst die von der Engine gemeldete Wiederverwendung aus dem nativen Präfix-Cache und dem Xavier-Cache. Sie ist Teil von ``prompt_tokens`` und keine zusätzliche Tokenanzahl." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "Setzen Sie bei Streaming-Anfragen ``stream_options={\"include_usage\": True}``, um den abschließenden Nutzungs-Chunk mit ``choices=[]`` zu erhalten. Die Responses API gibt dieselbe Anzahl als ``usage.input_tokens_details.cached_tokens`` zurück. Wenn die Engine keine Cache-Treffer meldet, lassen Completion- und Chat-Completion-Antworten ``prompt_tokens_details`` weg; die Responses API verwendet standardmäßig null." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "Bei der P/D-Trennung umfasst diese Anzahl auch vom Prefill-Motor übertragene KV, selbst wenn sie für die aktuelle Anfrage berechnet wurden; historische Präfix-Cache-Treffer im Prefill-Motor werden nicht separat gezählt." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po index ef239dd717..27aaad2eaa 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/client_api.po @@ -216,6 +216,10 @@ msgstr "Número de tokens encontrados en caché" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "Con vLLM, SGLang y MLX, las respuestas de completado y completado de chat exponen el número de tokens del prompt reutilizados por el motor en ``usage.prompt_tokens_details.cached_tokens``, cuando está disponible. Este número incluye la reutilización de la caché de prefijos nativa y de la caché Xavier que informa el motor; forma parte de ``prompt_tokens`` y no es un número adicional de tokens." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "Para las solicitudes en streaming, establezca ``stream_options={\"include_usage\": True}`` para recibir el bloque final de uso con ``choices=[]``. La Responses API expone el mismo número como ``usage.input_tokens_details.cached_tokens``. Si el motor no informa de aciertos de caché, las respuestas de completado y completado de chat omiten ``prompt_tokens_details``; la Responses API utiliza cero de forma predeterminada." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "Con la separación P/D, este número incluye los KV transferidos desde el motor de prellenado, aunque se hayan calculado para la solicitud actual; no mide por separado los aciertos históricos de la caché de prefijos del motor de prellenado." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po index 9c2a7978be..780f6d6e2b 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/client_api.po @@ -220,6 +220,10 @@ msgstr "Nombre de tokens trouvés en cache" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "Avec vLLM, SGLang et MLX, les réponses de complétion et de complétion de chat exposent le nombre de tokens du prompt réutilisés par le moteur dans ``usage.prompt_tokens_details.cached_tokens``, lorsque cette information est disponible. Ce nombre inclut la réutilisation du cache de préfixes natif et du cache Xavier signalée par le moteur ; il fait partie de ``prompt_tokens`` et ne constitue pas un nombre de tokens supplémentaire." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "Pour les requêtes en streaming, définissez ``stream_options={\"include_usage\": True}`` afin de recevoir le dernier bloc de statistiques avec ``choices=[]``. La Responses API expose le même nombre dans ``usage.input_tokens_details.cached_tokens``. Si le moteur ne signale pas les succès de cache, les réponses de complétion et de complétion de chat omettent ``prompt_tokens_details`` ; la Responses API utilise zéro par défaut." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "Avec la séparation P/D, ce nombre inclut les KV transférés depuis le moteur de préremplissage, même si ces KV ont été calculés pour la requête en cours ; il ne mesure pas séparément les succès du cache de préfixes historique de ce moteur." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po index b60662c29f..0a5b58de6f 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/client_api.po @@ -219,6 +219,10 @@ msgstr "Numero di token trovati nella cache" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "Con vLLM, SGLang e MLX, le risposte di completamento e completamento chat espongono il numero di token del prompt riutilizzati dal motore in ``usage.prompt_tokens_details.cached_tokens``, quando disponibile. Questo numero include il riutilizzo della cache dei prefissi nativa e della cache Xavier riportato dal motore; fa parte di ``prompt_tokens`` e non rappresenta un numero aggiuntivo di token." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "Per le richieste in streaming, impostare ``stream_options={\"include_usage\": True}`` per ricevere il blocco finale delle statistiche di utilizzo con ``choices=[]``. La Responses API espone lo stesso numero come ``usage.input_tokens_details.cached_tokens``. Se il motore non riporta gli accessi riusciti alla cache, le risposte di completamento e completamento chat omettono ``prompt_tokens_details``; la Responses API utilizza zero come valore predefinito." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "Con la separazione P/D, questo numero include i KV trasferiti dal motore di prefill, anche se sono stati calcolati per la richiesta corrente; non misura separatamente gli accessi alla cache dei prefissi di richieste precedenti nel motore di prefill." diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po index 8af276d337..989c59693c 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/client_api.po @@ -201,6 +201,10 @@ msgstr "キャッシュヒットしたトークン数" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "vLLM、SGLang、MLX では、エンジンが統計を提供する場合、補完およびチャット補完のレスポンスの ``usage.prompt_tokens_details.cached_tokens`` に、再利用されたプロンプトトークン数が含まれます。この数値には、エンジンが報告したネイティブのプレフィックスキャッシュと Xavier キャッシュの再利用が含まれます。これは ``prompt_tokens`` の一部であり、追加のトークン数ではありません。" -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "ストリーミングリクエストでは、``stream_options={\"include_usage\": True}`` を設定すると、``choices=[]`` の最終使用量チャンクを受け取れます。Responses API は同じ数値を ``usage.input_tokens_details.cached_tokens`` で返します。エンジンがキャッシュヒットを報告しない場合、補完およびチャット補完のレスポンスでは ``prompt_tokens_details`` が省略され、Responses API では既定値のゼロが返されます。" + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "P/D 分離では、この数値にプリフィルエンジンから転送された KV が含まれます。その KV が現在のリクエストで計算された場合も含まれ、プリフィルエンジンでの過去のプレフィックスキャッシュのヒット数を個別に示すものではありません。" diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po index 47b9b3b522..5654d2d73c 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/client_api.po @@ -202,6 +202,10 @@ msgstr "캐시 적중 토큰 수" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "vLLM, SGLang, MLX에서 엔진이 통계를 제공하면 완성 및 채팅 완성 응답의 ``usage.prompt_tokens_details.cached_tokens``에 재사용된 프롬프트 토큰 수가 포함됩니다. 이 값에는 엔진이 보고한 기본 접두사 캐시 재사용과 Xavier 캐시 재사용이 포함됩니다. ``prompt_tokens``의 일부이며 추가 토큰 수가 아닙니다." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "스트리밍 요청에서 ``stream_options={\"include_usage\": True}``를 설정하면 ``choices=[]``인 최종 사용량 청크를 받을 수 있습니다. Responses API는 동일한 값을 ``usage.input_tokens_details.cached_tokens``로 반환합니다. 엔진이 캐시 적중을 보고하지 않으면 완성 및 채팅 완성 응답에서 ``prompt_tokens_details``가 생략되고 Responses API는 기본값인 0을 반환합니다." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "P/D 분리에서는 현재 요청을 위해 계산된 KV도 포함하여 프리필 엔진에서 전송된 KV를 이 값에 포함합니다. 프리필 엔진의 이전 접두사 캐시 적중 횟수를 별도로 나타내지는 않습니다." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po index 2ea767c523..679d561ebb 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/client_api.po @@ -216,6 +216,10 @@ msgstr "Quantidade de tokens encontrados no cache" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "Com vLLM, SGLang e MLX, as respostas de completamento e completamento de chat expõem a quantidade de tokens do prompt reutilizados pelo mecanismo em ``usage.prompt_tokens_details.cached_tokens``, quando disponível. Essa quantidade inclui a reutilização do cache de prefixos nativo e do cache Xavier informada pelo mecanismo; faz parte de ``prompt_tokens`` e não é uma quantidade adicional de tokens." -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "Para solicitações em streaming, defina ``stream_options={\"include_usage\": True}`` para receber o bloco final de uso com ``choices=[]``. A Responses API expõe a mesma quantidade como ``usage.input_tokens_details.cached_tokens``. Se o mecanismo não informar acertos de cache, as respostas de completamento e completamento de chat omitem ``prompt_tokens_details``; a Responses API usa zero por padrão." + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "Com a separação P/D, essa quantidade inclui os KV transferidos do mecanismo de prefill, mesmo quando foram calculados para a solicitação atual; ela não mede separadamente os acertos históricos do cache de prefixos do mecanismo de prefill." diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po index 42e51c8012..b84d2af7aa 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/client_api.po @@ -236,6 +236,10 @@ msgstr "缓存命中 token 数" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "使用 vLLM、SGLang 和 MLX 时，如果引擎提供统计，补全和聊天补全响应会通过 ``usage.prompt_tokens_details.cached_tokens`` 返回复用的提示 token 数。该数值包含引擎报告的原生前缀缓存复用和 Xavier 缓存复用，是 ``prompt_tokens`` 的一部分，不是额外的 token 数。" -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "流式请求设置 ``stream_options={\"include_usage\": True}`` 后，会收到 ``choices=[]`` 的最终用量统计块。Responses API 通过 ``usage.input_tokens_details.cached_tokens`` 返回同一数值。如果引擎未报告缓存命中，补全和聊天补全响应会省略 ``prompt_tokens_details``；Responses API 则默认返回零。" + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "使用 P/D 分离时，该数值包含从预填充引擎传输的 KV，即使这些 KV 是为当前请求计算的；它不单独统计预填充引擎的历史前缀缓存命中数。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po index cf1100d04d..ddd221cbbd 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/client_api.po @@ -197,6 +197,10 @@ msgstr "快取命中 token 數" msgid "With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count." msgstr "使用 vLLM、SGLang 和 MLX 時，如果引擎提供統計，補全和聊天補全回應會透過 ``usage.prompt_tokens_details.cached_tokens`` 傳回重用的提示 token 數。此數值包含引擎回報的原生前綴快取重用和 Xavier 快取重用，是 ``prompt_tokens`` 的一部分，不是額外的 token 數。" -#: user_guide/client_api.rst:101 +#: user_guide/client_api.rst:103 msgid "For streaming requests, set ``stream_options={\"include_usage\": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero." msgstr "串流請求設定 ``stream_options={\"include_usage\": True}`` 後，會收到 ``choices=[]`` 的最終用量統計區塊。Responses API 透過 ``usage.input_tokens_details.cached_tokens`` 傳回相同數值。如果引擎未回報快取命中，補全和聊天補全回應會省略 ``prompt_tokens_details``；Responses API 則預設傳回零。" + +#: user_guide/client_api.rst:101 +msgid "With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine." +msgstr "使用 P/D 分離時，此數值包含從預填充引擎傳輸的 KV，即使這些 KV 是為目前請求計算的；它不單獨統計預填充引擎的歷史前綴快取命中數。" diff --git a/doc/source/user_guide/client_api.rst b/doc/source/user_guide/client_api.rst index 9b2ab364d4..4d242b176a 100644 --- a/doc/source/user_guide/client_api.rst +++ b/doc/source/user_guide/client_api.rst @@ -98,6 +98,8 @@ Cache hit token counts With vLLM, SGLang, and MLX, completion and chat completion responses expose the engine's reused prompt-token count in ``usage.prompt_tokens_details.cached_tokens`` when available. This count includes native prefix-cache reuse and Xavier cache reuse reported by the engine; it is part of ``prompt_tokens``, not an additional token count. +With P/D disaggregation, this count includes KV transferred from the prefill engine, even when that KV was computed for the current request; it does not separately measure historical prefix-cache hits on the prefill engine. + For streaming requests, set ``stream_options={"include_usage": True}`` to receive the final usage chunk with ``choices=[]``. The Responses API exposes the same count as ``usage.input_tokens_details.cached_tokens``. If the engine does not report cache hits, completion and chat completion responses omit ``prompt_tokens_details``; the Responses API defaults the count to zero. OpenAI Client Tool Calls diff --git a/xinference/model/llm/tests/test_cached_token_usage.py b/xinference/model/llm/tests/test_cached_token_usage.py index 32a9d9c00d..b8dda0f743 100644 --- a/xinference/model/llm/tests/test_cached_token_usage.py +++ b/xinference/model/llm/tests/test_cached_token_usage.py @@ -38,9 +38,14 @@ def _usage(cached_tokens): @pytest.fixture(params=["vllm", "sglang", "mlx"]) -def model_factory(request, monkeypatch): +def engine(request): + return request.param + + +@pytest.fixture +def model_factory(engine, monkeypatch): def create(cached_tokens): - if request.param == "vllm": + if engine == "vllm": from ..vllm import core class SamplingParams: @@ -83,7 +88,7 @@ async def generate(*args, **kwargs): model.async_generate = core.VLLMModel.async_generate.__wrapped__.__get__( model ) - elif request.param == "sglang": + elif engine == "sglang": from ..sglang.core import SGLANGModel model = object.__new__(SGLANGModel) @@ -209,6 +214,7 @@ async def source(): @pytest.mark.asyncio +@pytest.mark.parametrize("engine", ["vllm", "sglang"]) @pytest.mark.parametrize("cached_tokens", [MISSING, None]) @pytest.mark.parametrize("stream", [False, True]) async def test_unreported_cached_tokens_remain_optional( @@ -226,8 +232,96 @@ async def test_unreported_cached_tokens_remain_optional( chunks = [chunk async for chunk in result] assert chunks[-1]["choices"] == [] assert chunks[-1]["usage"] == _usage(cached_tokens) + + async def source(): + for chunk in chunks: + yield chunk + + events = [ + json.loads(event["data"]) + async for event in responses_stream_events( + ChatModelMixin._async_to_chat_completion_chunks(source()), + parse_responses_request( + {"model": "test-model", "input": "prompt", "stream": True} + ), + ) + ] + response = events[-1]["response"] else: assert result["usage"] == _usage(cached_tokens) + response = chat_to_response( + ChatModelMixin._to_chat_completion(result), + parse_responses_request({"model": "test-model", "input": "prompt"}), + ) + assert response["usage"]["input_tokens_details"] == {"cached_tokens": 0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cached_tokens", [0, 64]) +async def test_tool_stream_preserves_cached_tokens_in_usage_chunk( + model_factory, cached_tokens +): + from ..tool_parsers.qwen_tool_parser import QwenToolParser + + completions = await model_factory(cached_tokens).async_generate( + "prompt", + {"max_tokens": 2, "stream": True, "stream_options": {"include_usage": True}}, + ) + + async def tool_completions(): + sent = False + async for chunk in completions: + for choice in chunk["choices"]: + choice["text"] = ( + '{"name":"get_weather","arguments":{"city":"Beijing"}}' + "" + if not sent + else "" + ) + sent = True + yield chunk + + mixin = ChatModelMixin() + mixin.model_family = "qwen2-instruct" + mixin.model_uid = "test-model" + mixin.reasoning_parser = None + mixin.tool_parser = QwenToolParser() + chunks = [ + chunk + async for chunk in mixin._async_to_tool_completion_chunks(tool_completions()) + ] + final = next( + chunk + for chunk in reversed(chunks) + if chunk["choices"] and chunk["choices"][0]["finish_reason"] + ) + assert final["choices"][0]["finish_reason"] == "tool_calls" + assert final["usage"] is None + assert any( + call["function"].get("name") == "get_weather" + for chunk in chunks + for choice in chunk["choices"] + for call in choice["delta"].get("tool_calls", []) + ) + assert chunks[-1]["choices"] == [] + assert chunks[-1]["usage"] == _usage(cached_tokens) + + async def source(): + for chunk in chunks: + yield chunk + + events = [ + json.loads(event["data"]) + async for event in responses_stream_events( + source(), + parse_responses_request( + {"model": "test-model", "input": "prompt", "stream": True} + ), + ) + ] + assert events[-1]["response"]["usage"]["input_tokens_details"] == { + "cached_tokens": cached_tokens + } def test_vllm_multiple_outputs_do_not_multiply_cached_tokens():