From 3aec5f39c7ca4b05921ab021583f8e88441fe0b1 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Mon, 5 Oct 2026 23:51:08 +0800 Subject: [PATCH 1/4] feat: support MLX Xavier cache sharing and PD separation --- .github/workflows/python.yaml | 1 + benchmark/README.md | 20 ++ benchmark/benchmark_mlx_xavier.py | 304 ++++++++++++++++ .../LC_MESSAGES/user_guide/pd_separation.po | 70 ++++ .../user_guide/vllm_enhancement.po | 31 +- .../LC_MESSAGES/user_guide/pd_separation.po | 69 ++++ .../user_guide/vllm_enhancement.po | 32 +- .../LC_MESSAGES/user_guide/pd_separation.po | 71 ++++ .../user_guide/vllm_enhancement.po | 32 +- .../LC_MESSAGES/user_guide/pd_separation.po | 69 ++++ .../user_guide/vllm_enhancement.po | 32 +- .../LC_MESSAGES/user_guide/pd_separation.po | 60 ++++ .../user_guide/vllm_enhancement.po | 28 +- .../LC_MESSAGES/user_guide/pd_separation.po | 59 ++++ .../user_guide/vllm_enhancement.po | 28 +- .../LC_MESSAGES/user_guide/pd_separation.po | 67 ++++ .../user_guide/vllm_enhancement.po | 31 +- .../LC_MESSAGES/user_guide/pd_separation.po | 56 +++ .../user_guide/vllm_enhancement.po | 27 +- .../LC_MESSAGES/user_guide/pd_separation.po | 56 +++ .../user_guide/vllm_enhancement.po | 27 +- doc/source/user_guide/pd_separation.rst | 30 ++ doc/source/user_guide/vllm_enhancement.rst | 7 +- .../replica-placement-config.tsx | 2 +- xinference/core/pd_model.py | 8 +- xinference/core/replica_config.py | 4 +- xinference/core/supervisor.py | 46 ++- xinference/core/tests/test_pd_launch.py | 70 ++++ xinference/core/tests/test_pd_model.py | 11 +- xinference/model/llm/mlx/core.py | 146 +++++++- xinference/model/llm/mlx/tests/test_xavier.py | 312 ++++++++++++++++ xinference/model/llm/mlx/xavier.py | 334 ++++++++++++++++++ .../llm/xavier/backends/bytes/__init__.py | 2 + .../llm/xavier/backends/bytes/storage.py | 157 ++++++++ .../llm/xavier/tests/test_bytes_storage.py | 83 +++++ 35 files changed, 2343 insertions(+), 39 deletions(-) create mode 100644 benchmark/benchmark_mlx_xavier.py create mode 100644 xinference/model/llm/mlx/tests/test_xavier.py create mode 100644 xinference/model/llm/mlx/xavier.py create mode 100644 xinference/model/llm/xavier/backends/bytes/__init__.py create mode 100644 xinference/model/llm/xavier/backends/bytes/storage.py create mode 100644 xinference/model/llm/xavier/tests/test_bytes_storage.py diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 38539b5335..3d9fed2c1e 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -347,6 +347,7 @@ jobs: python -m pip freeze > test-results/metal-host-packages.txt for test_file in \ xinference/model/llm/mlx/tests/test_mlx.py \ + xinference/model/llm/mlx/tests/test_xavier.py \ xinference/model/audio/tests/test_whisper_mlx.py \ xinference/model/audio/tests/test_f5tts_mlx.py \ xinference/model/llm/mlx/tests/test_distributed_model.py; do diff --git a/benchmark/README.md b/benchmark/README.md index a2fc7cc9bc..adc06da8a1 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -138,7 +138,27 @@ python benchmark/benchmark_long.py --context-length ${context_length} --tokenize --num-prompts 32 -c 16 ``` +## MLX Xavier cache and P/D + +On Apple silicon with `mlx-lm>=0.31.2`, compare ordinary MLX replicas, shared +Xavier, and Xavier P/D through the same OpenAI API: + +```bash +python benchmark/benchmark_mlx_xavier.py \ + --model-path /path/to/fp16/Qwen2.5-0.5B-Instruct \ + --output artifacts/mlx-xavier --requests 8 +``` + +Use unquantized **FP16** safetensors for all modes. Xavier converts weights to +FP16; using BF16 for ordinary replicas would change the comparison. Each mode +starts a private server with two independent model processes and keeps the +default local prompt cache enabled. The script records cold and repeated-prompt +TTFT, latency, output throughput, actual cached tokens, greedy output equality, +and shared-cache reads/leases. These are sequential requests on one shared Metal +GPU, not a multi-host or NVIDIA-to-Mac network benchmark. + ## Common Options for Benchmarking Tools + - `--stream`. You can enable streaming responses by using the option, which is useful for real-time data processing and receiving incremental data without waiting for the entire dataset to be processed. - `--print-error`. For troubleshooting and more detailed output, the option can be used to print detailed error messages if any errors are encountered during the execution. diff --git a/benchmark/benchmark_mlx_xavier.py b/benchmark/benchmark_mlx_xavier.py new file mode 100644 index 0000000000..2fee9f7147 --- /dev/null +++ b/benchmark/benchmark_mlx_xavier.py @@ -0,0 +1,304 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""Compare ordinary replicas, shared Xavier, and Xavier P/D on one Apple GPU. + +Use a local unquantized FP16 Qwen2.5-0.5B-Instruct checkpoint. Each mode starts +its own Xinference server and two independent MLX model processes. This measures +API overhead and cache reuse on one Mac, not multi-host network performance. +""" + +import argparse +import asyncio +import hashlib +import importlib.metadata +import json +import os +import platform +import re +import signal +import subprocess +import sys +import time +from pathlib import Path + +import psutil +import requests +import xoscar as xo +from openai import AsyncOpenAI +from safetensors import safe_open +from xoscar.utils import get_next_port + +from xinference.client import Client + + +async def measure(endpoint, uid, messages, count, max_tokens): + records = [] + async with AsyncOpenAI( + base_url=endpoint + "/v1", api_key="unused", timeout=180, max_retries=0 + ) as api: + for _ in range(count): + start = time.perf_counter() + first, usage, text = None, None, [] + stream = await api.chat.completions.create( + model=uid, + messages=messages, + temperature=0, + max_tokens=max_tokens, + stream=True, + stream_options={"include_usage": True}, + ) + async with stream: + async for chunk in stream: + if chunk.usage: + usage = chunk.usage + if chunk.choices and chunk.choices[0].delta.content: + first = first or time.perf_counter() + text.append(chunk.choices[0].delta.content) + if first is None or usage is None: + raise RuntimeError("Missing streamed text or token usage") + duration = time.perf_counter() - start + records.append( + dict( + ttft_ms=(first - start) * 1000, + latency_ms=duration * 1000, + output_tps=usage.completion_tokens / duration, + prompt_tokens=usage.prompt_tokens, + cached_tokens=( + usage.prompt_tokens_details.cached_tokens + if usage.prompt_tokens_details + else 0 + ), + output_tokens=usage.completion_tokens, + text="".join(text), + ) + ) + return records + + +def stop_server(proc): + # Only terminate the server and descendants created by this benchmark. + try: + children = psutil.Process(proc.pid).children(recursive=True) + except psutil.NoSuchProcess: + children = [] + if proc.poll() is None: + os.killpg(proc.pid, signal.SIGTERM) + for child in reversed(children): + try: + child.terminate() + except psutil.NoSuchProcess: + pass + try: + proc.wait(timeout=20) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + _, alive = psutil.wait_procs(children, timeout=10) + for child in alive: + child.kill() + + +async def stats(address, uid): + ref = await xo.actor_ref(address=address, uid=f"xavier-cache-{uid}") + return await ref.get_stats() + + +def run_mode(args, mode): + root = args.output / mode + root.mkdir(parents=True, exist_ok=True) + endpoint = f"http://127.0.0.1:{get_next_port()}" + uid = f"mlx-{mode}" + env = dict( + os.environ, + PYTHONPATH=str(Path(__file__).resolve().parents[1]), + XINFERENCE_HOME=str(root / "home"), + XINFERENCE_ENABLE_VIRTUAL_ENV="0", + XINFERENCE_AUTH_ADVANCED="false", + ) + with (root / "server.log").open("w") as log: + proc = subprocess.Popen( + [ + sys.executable, + "-c", + "from xinference.deploy.cmdline import local; local()", + "--host", + "127.0.0.1", + "--port", + endpoint.rsplit(":", 1)[1], + ], + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + client, launched = None, False + try: + for _ in range(120): + if proc.poll() is not None: + raise RuntimeError(f"Server exited; inspect {root / 'server.log'}") + try: + client = Client(endpoint) + workers = client.get_workers_info() + if workers: + break + except requests.RequestException: + pass + time.sleep(1) + else: + raise RuntimeError("Server readiness timed out") + launch = dict( + model_uid=uid, + model_name="qwen2.5-instruct", + model_size_in_billions="0_5", + model_engine="MLX", + model_format="mlx", + quantization="none", + model_path=str(args.model_path), + replica=2, + enable_virtual_env=False, + prompt_cache_size=2, + batch_size=4, + ) + if mode == "shared": + launch["enable_xavier"] = True + elif mode == "pd": + launch["replica_config"] = [ + dict( + role=role, + devices=[dict(worker_ip=workers[0]["work-ip"], n_gpu="auto")], + ) + for role in ("prefill", "decode") + ] + (root / "launch.json").write_text(json.dumps(launch, indent=2)) + client.launch_model(**launch) + launched = True + # Prime kernels with an unrelated short prompt, leaving measured + # document prefixes cold. Both ordinary replicas receive warm-up. + asyncio.run( + measure( + endpoint, uid, [{"role": "user", "content": "Say hello."}], 2, 8 + ) + ) + messages = [ + dict( + role="user", + content=( + "Document: " + + "Water evaporates in sunlight and condenses into clouds. " + * args.repeats + + " Explain this process in detail." + ), + ) + ] + cold = asyncio.run(measure(endpoint, uid, messages, 1, args.max_tokens)) + warm = asyncio.run( + measure(endpoint, uid, messages, args.requests, args.max_tokens) + ) + response = requests.post( + endpoint + "/v1/chat/completions", + json=dict( + model=uid, + messages=messages, + temperature=0, + max_tokens=args.max_tokens, + ), + timeout=180, + ) + response.raise_for_status() + nonstream = response.json() + processes = [ + dict(pid=int(pid), model_uid=model_uid) + for pid, model_uid in re.findall( + r"pid:(\d+).*?ModelActor\(([^)]+)\) loaded", + (root / "server.log").read_text(), + ) + ] + if len({p["pid"] for p in processes}) != 2: + raise RuntimeError("Expected two independent MLX model processes") + result = dict( + mode=mode, + launch=launch, + messages=messages, + cold=cold, + warm=warm, + nonstream=nonstream, + cache_stats=( + asyncio.run(stats(workers[0]["work-ip"], uid)) + if mode != "ordinary" + else None + ), + model_processes=processes, + ) + if mode != "ordinary": + cache = result["cache_stats"] + if cache["active_handoffs"] or not cache["read_pages"]: + raise RuntimeError("Missing cache reads or leaked handoffs") + if any(r["cached_tokens"] != r["prompt_tokens"] - 1 for r in warm): + raise RuntimeError("Warm requests did not reuse their full prefix") + (root / "results.json").write_text(json.dumps(result, indent=2)) + print(mode, json.dumps(result["cold"] + result["warm"]), flush=True) + return result + finally: + if launched: + try: + client.terminate_model(uid) + except Exception: + pass + stop_server(proc) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-path", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--requests", type=int, default=8) + parser.add_argument("--repeats", type=int, default=100) + parser.add_argument("--max-tokens", type=int, default=32) + args = parser.parse_args() + if min(args.requests, args.repeats, args.max_tokens) <= 0: + parser.error("requests, repeats and max-tokens must be positive") + args.model_path = args.model_path.resolve() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + weights = list(args.model_path.glob("*.safetensors")) + if not weights: + parser.error("model-path must contain FP16 safetensors") + for file in weights: + with safe_open(file, framework="np") as checkpoint: + if any( + checkpoint.get_slice(key).get_dtype() != "F16" + for key in checkpoint.keys() + ): + parser.error("all modes require an unquantized FP16 checkpoint") + result = dict( + platform=platform.platform(), + versions={ + name: importlib.metadata.version(name) + for name in ("mlx", "mlx-lm", "xoscar") + }, + revision=subprocess.check_output( + ["git", "rev-parse", "HEAD"], text=True + ).strip(), + diff_sha256=hashlib.sha256( + subprocess.check_output(["git", "diff", "HEAD"]) + ).hexdigest(), + note="One Mac with a shared Metal GPU; no multi-host or NVIDIA-to-Mac measurements.", + modes=[], + ) + for mode in ("ordinary", "shared", "pd"): + result["modes"].append(run_mode(args, mode)) + (args.output / "results.json").write_text(json.dumps(result, indent=2)) + texts = [ + row["text"] for mode in result["modes"] for row in mode["cold"] + mode["warm"] + ] + [ + mode["nonstream"]["choices"][0]["message"]["content"] + for mode in result["modes"] + ] + result["all_texts_match"] = len(set(texts)) == 1 + (args.output / "results.json").write_text(json.dumps(result, indent=2)) + if not result["all_texts_match"]: + raise RuntimeError("Greedy outputs differ; inspect the per-request records") + + +if __name__ == "__main__": + main() diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po index 0feb4130b4..6153d5ac3c 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po @@ -441,3 +441,73 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "Natives SGLang P/D erfordert einen einzelnen Tokenizer-Worker; interne HTTP-Endpunkte verwenden automatisch eine Authentifizierung pro Replikat." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX mit Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier benötigt Apple silicon und ``mlx-lm>=0.31.2``. Unterstützt werden" +" unquantisierte Qwen2-, Qwen3- und Llama-Textmodelle mit vollständiger " +"Attention und einem Worker pro Replikat. Gewichte und KV-Cache verwenden " +"FP16. Quantisierte Gewichte oder KV-Caches, rotierende Caches, hybride " +"Attention, multimodale Eingaben, LoRA und spekulatives Decoding werden nicht" +" unterstützt." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"Verwenden Sie ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` und " +"``quantization=\"none\"`` mit denselben Prefill-/Decode-Rollen. Das Beispiel" +" platziert je ein Replikat auf zwei Mac-Workern. Ersetzen Sie die " +"registrierten Worker-Adressen durch die Werte von " +"``client.get_workers_info()``." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier speichert unveränderliche CPU-Seiten in einem gemeinsamen Actor " +"auf dem Supervisor und überträgt sie per Actor-RPC. ``xavier_cache_bytes`` " +"begrenzt diesen Deployment-Cache (Standard: 512 MiB); Modellgewichte und " +"Metal-KV-Speicher jedes Replikats benötigen zusätzlichen Speicher. Prefill " +"berechnet alle Prompt-Token außer dem letzten. Decode importiert das Präfix," +" berechnet das letzte Token und wählt das erste Antwort-Token. Streaming- " +"und Nicht-Streaming-Anfragen werden unterstützt. Aktive Übergaben " +"reservieren Cache-Kapazität; unzureichende Kapazität, inkompatible Metadaten" +" oder Übertragungsfehler lösen einen Fehler aus. Nicht abgeholte Übergaben " +"verfallen nach fünf Minuten." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"Verwenden Sie für MLX den Standardtransport ``xavier``; natives NIXL wird " +"nicht unterstützt. Alle Worker müssen die Actor-Adresse des Supervisors " +"erreichen. Starten Sie das Deployment nach einem Worker-Neustart neu. Zwei " +"Replikate auf einem Mac teilen dessen Metal-GPU; messen Sie Ihren Workload, " +"bevor Sie einen Durchsatzvorteil erwarten. Engineübergreifendes P/D " +"einschließlich NVIDIA-zu-Mac-Übergaben wird noch nicht unterstützt." diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/de/LC_MESSAGES/user_guide/vllm_enhancement.po index f5d9c3c424..54ab14e090 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -75,8 +75,8 @@ msgstr "" "konfiguriert werden, z. B.: ``xinference-local -H 192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier unterstützt nur Nvidia-Grafikkarten." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "Der vLLM-GPU-Transferpfad benötigt NVIDIA-GPUs." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -103,3 +103,30 @@ msgstr "Gewöhnliche SGLang-Replikate teilen CPU-HiCache-Seiten. Explizite prefi #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier lädt BF16-Checkpoints als FP16 und gibt eine Warnung aus. Sowohl Gewichte als auch KV-Caches verwenden FP16." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "MLX-Replikate" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"MLX-Replikate können mit ``model_engine=\"MLX\"`` und ``enable_xavier=True``" +" Präfix-KV-Caches teilen. Unterstützte Modelle und FP16-Anforderungen stehen" +" unter :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` begrenzt den " +"gemeinsamen CPU-Cache (Standard: 512 MiB). Jedes Replikat besitzt zusätzlich" +" sein Modell und seinen Metal-KV-Cache. Gewöhnliche Replikate rechnen bei " +"einem Cache-Miss oder nicht verfügbarem Speicher lokal; inkompatible " +"Deployments können keine Seiten teilen. Die Nutzungsdaten melden " +"wiederverwendete Token unter ``prompt_tokens_details.cached_tokens``. " +"Explizite Prefill-/Decode-Rollen aktivieren MLX Xavier P/D automatisch. " +"Engineübergreifendes Cache-Sharing wird noch nicht unterstützt." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po index a1a8bcd7fa..5b1fd16b86 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po @@ -437,3 +437,72 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "El P/D nativo de SGLang requiere un único worker de tokenización; sus endpoints HTTP internos usan automáticamente autenticación por réplica." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX con Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier requiere Apple silicon y ``mlx-lm>=0.31.2``. Admite modelos de " +"texto Qwen2, Qwen3 y Llama sin cuantizar, con atención completa y un worker " +"por réplica. Los pesos y la caché KV usan FP16. No admite pesos o cachés KV " +"cuantizados, cachés rotativas, atención híbrida, entradas multimodales, LoRA" +" ni decodificación especulativa." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` y " +"``quantization=\"none\"`` con los mismos roles prefill/decode. El ejemplo " +"coloca una réplica en cada uno de dos workers Mac. Sustituya las direcciones" +" de los workers registrados por las devueltas por " +"``client.get_workers_info()``." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier guarda páginas CPU inmutables en un actor compartido del " +"supervisor y las transfiere por RPC de actores. ``xavier_cache_bytes`` " +"limita esta caché del despliegue (512 MiB por defecto); los pesos y las " +"asignaciones KV Metal de cada réplica requieren memoria adicional. Prefill " +"calcula todos los tokens del prompt salvo el último. Decode importa el " +"prefijo, calcula ese último token y muestrea el primer token de respuesta. " +"Se admiten solicitudes con y sin streaming. Las transferencias activas " +"reservan capacidad de caché; capacidad insuficiente, metadatos incompatibles" +" o fallos de transferencia generan un error. Las transferencias no recogidas" +" caducan a los cinco minutos." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"Use el transporte predeterminado ``xavier`` para MLX; NIXL nativo no es " +"compatible. Todos los workers deben acceder a la dirección actor del " +"supervisor. Reinicie el despliegue tras reiniciar un worker. Dos réplicas en" +" un Mac comparten su GPU Metal; mida su carga antes de esperar mejoras de " +"rendimiento. El P/D entre motores, incluida la transferencia de NVIDIA a " +"Mac, aún no es compatible." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/es/LC_MESSAGES/user_guide/vllm_enhancement.po index 01d456b974..0ff9050dd5 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -75,8 +75,8 @@ msgstr "" "real, por ejemplo: ``xinference-local -H 192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier solo es compatible con tarjetas gráficas Nvidia." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "La transferencia GPU de vLLM requiere GPU NVIDIA." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -103,3 +103,31 @@ msgstr "Las réplicas SGLang normales comparten páginas CPU HiCache. Los roles #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier carga los puntos de control BF16 como FP16 y emite una advertencia. Tanto los pesos como las cachés KV usan FP16." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "Réplicas MLX" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"Las réplicas MLX pueden compartir la caché KV de prefijos con " +"``model_engine=\"MLX\"`` y ``enable_xavier=True``. Los modelos admitidos y " +"requisitos FP16 se describen en :ref:`user_guide_pd_separation`. " +"``xavier_cache_bytes`` limita la caché CPU compartida (512 MiB por defecto)." +" Cada réplica también tiene su modelo y caché KV Metal. Las réplicas " +"ordinarias calculan localmente si no hay acierto o el almacenamiento no está" +" disponible; los despliegues incompatibles no pueden compartir páginas. El " +"uso informa los tokens reutilizados en " +"``prompt_tokens_details.cached_tokens``. Los roles prefill/decode explícitos" +" activan MLX Xavier P/D automáticamente. Aún no se admite compartir caché " +"entre motores." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po index 2ece9549dc..65fb7b975c 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po @@ -442,3 +442,74 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "Le P/D natif SGLang nécessite un seul worker de tokenisation ; ses endpoints HTTP internes utilisent automatiquement une authentification propre à chaque réplique." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX avec Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier nécessite Apple silicon et ``mlx-lm>=0.31.2``. Il prend en charge" +" les modèles texte Qwen2, Qwen3 et Llama non quantifiés, à attention " +"complète, avec un worker par réplique. Les poids et le cache KV utilisent " +"FP16. Les poids ou caches KV quantifiés, les caches rotatifs, l’attention " +"hybride, les entrées multimodales, LoRA et le décodage spéculatif ne sont " +"pas pris en charge." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"Utilisez ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` et " +"``quantization=\"none\"`` avec les mêmes rôles prefill/decode. L’exemple " +"place une réplique sur chacun de deux workers Mac. Remplacez les adresses " +"des workers enregistrés par celles renvoyées par " +"``client.get_workers_info()``." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier stocke des pages CPU immuables dans un actor partagé du " +"supervisor et les transfère par RPC actor. ``xavier_cache_bytes`` limite ce " +"cache de déploiement (512 MiB par défaut) ; les poids du modèle et les " +"allocations KV Metal de chaque réplique demandent de la mémoire " +"supplémentaire. Prefill calcule tous les tokens du prompt sauf le dernier. " +"Decode importe le préfixe, calcule ce dernier token et échantillonne le " +"premier token de réponse. Les requêtes avec ou sans streaming sont prises en" +" charge. Les transferts actifs réservent la capacité du cache ; une capacité" +" insuffisante, des métadonnées incompatibles ou un échec de transfert " +"provoquent une erreur. Les transferts non récupérés expirent après cinq " +"minutes." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"Utilisez le transport par défaut ``xavier`` pour MLX ; NIXL natif n’est pas " +"pris en charge. Tous les workers doivent pouvoir joindre l’adresse actor du " +"supervisor. Relancez le déploiement après le redémarrage d’un worker. Deux " +"répliques sur un même Mac partagent son GPU Metal ; mesurez votre charge " +"avant d’attendre un gain de débit. Le P/D entre moteurs, notamment de NVIDIA" +" vers Mac, n’est pas encore pris en charge." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/vllm_enhancement.po index 9e7da27fa6..92a0e2ad09 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -78,8 +78,8 @@ msgstr "" "192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier ne prend en charge que les cartes graphiques Nvidia." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "Le transfert GPU de vLLM nécessite des GPU NVIDIA." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -106,3 +106,31 @@ msgstr "Les répliques SGLang ordinaires partagent des pages CPU HiCache. Les r #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier charge les checkpoints BF16 en FP16 et émet un avertissement. Les poids et les caches KV utilisent FP16." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "Répliques MLX" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"Les répliques MLX peuvent partager le cache KV de préfixe avec " +"``model_engine=\"MLX\"`` et ``enable_xavier=True``. Les modèles et exigences" +" FP16 sont décrits dans :ref:`user_guide_pd_separation`. " +"``xavier_cache_bytes`` limite le cache CPU partagé (512 MiB par défaut). " +"Chaque réplique possède aussi son modèle et son cache KV Metal. Les " +"répliques ordinaires calculent localement en cas d’absence de cache ou de " +"stockage indisponible ; les déploiements incompatibles ne peuvent pas " +"partager de pages. Les statistiques signalent les tokens réutilisés dans " +"``prompt_tokens_details.cached_tokens``. Les rôles prefill/decode explicites" +" activent automatiquement MLX Xavier P/D. Le partage de cache entre moteurs " +"n’est pas encore pris en charge." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po index 7b3e8398a4..2c16f8d1b6 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po @@ -439,3 +439,72 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "Il P/D nativo di SGLang richiede un solo worker di tokenizzazione; gli endpoint HTTP interni usano automaticamente un’autenticazione per replica." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX con Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier richiede Apple silicon e ``mlx-lm>=0.31.2``. Supporta modelli di " +"testo Qwen2, Qwen3 e Llama non quantizzati, con attenzione completa e un " +"worker per replica. Pesi e cache KV usano FP16. Non sono supportati pesi o " +"cache KV quantizzati, cache rotanti, attenzione ibrida, input multimodali, " +"LoRA e decodifica speculativa." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"Usare ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` e " +"``quantization=\"none\"`` con gli stessi ruoli prefill/decode. L’esempio " +"colloca una replica su ciascuno di due worker Mac. Sostituire gli indirizzi " +"dei worker registrati con quelli restituiti da " +"``client.get_workers_info()``." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier conserva pagine CPU immutabili in un actor condiviso sul " +"supervisor e le trasferisce tramite RPC degli actor. ``xavier_cache_bytes`` " +"limita la cache del deployment (512 MiB predefiniti); i pesi del modello e " +"le allocazioni KV Metal di ogni replica richiedono memoria aggiuntiva. " +"Prefill calcola tutti i token del prompt tranne l’ultimo. Decode importa il " +"prefisso, calcola l’ultimo token e campiona il primo token di risposta. Sono" +" supportate richieste con e senza streaming. I trasferimenti attivi " +"riservano capacità della cache; capacità insufficiente, metadati " +"incompatibili o errori di trasferimento generano un errore. I trasferimenti " +"non ritirati scadono dopo cinque minuti." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"Usare il trasporto predefinito ``xavier`` per MLX; NIXL nativo non è " +"supportato. Tutti i worker devono raggiungere l’indirizzo actor del " +"supervisor. Riavviare il deployment dopo il riavvio di un worker. Due " +"repliche sullo stesso Mac condividono la GPU Metal; misurare il carico prima" +" di aspettarsi un aumento del throughput. Il P/D tra motori, incluso il " +"trasferimento da NVIDIA a Mac, non è ancora supportato." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/it/LC_MESSAGES/user_guide/vllm_enhancement.po index d188013c1c..a1d87ec162 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -75,8 +75,8 @@ msgstr "" "esempio: ``xinference-local -H 192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier supporta solo schede grafiche Nvidia." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "Il trasferimento GPU di vLLM richiede GPU NVIDIA." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -103,3 +103,31 @@ msgstr "Le repliche SGLang ordinarie condividono pagine CPU HiCache. I ruoli pre #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier carica i checkpoint BF16 come FP16 ed emette un avviso. Sia i pesi sia le cache KV usano FP16." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "Repliche MLX" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"Le repliche MLX possono condividere la cache KV dei prefissi con " +"``model_engine=\"MLX\"`` e ``enable_xavier=True``. Modelli supportati e " +"requisiti FP16 sono descritti in :ref:`user_guide_pd_separation`. " +"``xavier_cache_bytes`` limita la cache CPU condivisa (512 MiB predefiniti). " +"Ogni replica possiede anche il proprio modello e la propria cache KV Metal. " +"Le repliche ordinarie calcolano localmente in caso di cache miss o storage " +"non disponibile; deployment incompatibili non possono condividere pagine. " +"L’utilizzo riporta i token riusati in " +"``prompt_tokens_details.cached_tokens``. I ruoli prefill/decode espliciti " +"attivano automaticamente MLX Xavier P/D. La condivisione della cache tra " +"motori non è ancora supportata." diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po index db648e6261..728ee91889 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po @@ -398,3 +398,63 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "ネイティブ SGLang P/D には単一の tokenizer worker が必要です。内部 HTTP エンドポイントにはレプリカごとの認証が自動的に設定されます。" + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "Xavier を使用する MLX" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier には Apple silicon と ``mlx-lm>=0.31.2`` が必要です。量子化されていない全注意の " +"Qwen2、Qwen3、Llama テキストモデルを、レプリカごとに 1 worker でサポートします。重みと KV キャッシュには FP16 " +"を使用します。量子化された重みや KV キャッシュ、回転キャッシュ、ハイブリッド注意、マルチモーダル入力、LoRA、投機的デコードは未対応です。" + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"同じ prefill/decode ロールで " +"``model_engine=\"MLX\"``、``model_format=\"mlx\"``、``quantization=\"none\"`` " +"を指定します。この例は 2 台の Mac worker にレプリカを 1 つずつ配置します。登録済み worker のアドレスを " +"``client.get_workers_info()`` の結果に置き換えてください。" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier は supervisor の共有 actor に不変の CPU ページを保存し、actor RPC " +"で転送します。``xavier_cache_bytes`` はデプロイ全体のキャッシュ容量を制限します（既定値 512 " +"MiB）。モデルの重みと各レプリカの Metal KV 割り当てには追加メモリが必要です。Prefill " +"はプロンプトの最後のトークン以外を計算します。Decode " +"はプレフィックスをインポートし、最後のトークンを計算して最初の応答トークンをサンプリングします。ストリーミングと非ストリーミングに対応します。実行中の引き継ぎはキャッシュ容量を予約し、容量不足、メタデータ不一致、転送失敗はエラーになります。未取得の引き継ぎは" +" 5 分で期限切れになります。" + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"MLX では既定の ``xavier`` 転送を使用します。ネイティブ NIXL は未対応です。全 worker から supervisor の " +"actor アドレスに接続できる必要があります。Worker 再起動後はデプロイを再起動してください。同じ Mac 上の 2 レプリカは Metal " +"GPU を共有するため、スループット改善は実際の負荷で測定してください。NVIDIA から Mac への引き継ぎを含む異なるエンジン間の P/D " +"は未対応です。" diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/vllm_enhancement.po index 74164a7b88..89028d5b22 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -66,8 +66,8 @@ msgstr "" "を起動する際には実際のIPアドレスを設定する必要があります。例：`xinference-local -H 192.168.xx.xx`。" #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "XavierはNvidiaのグラフィックカードのみをサポートしています。" +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "vLLM の GPU 転送には NVIDIA GPU が必要です。" #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -94,3 +94,27 @@ msgstr "通常の SGLang レプリカは CPU HiCache ページを共有します #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier は BF16 チェックポイントを FP16 として読み込み、警告を表示します。重みと KV キャッシュの両方に FP16 を使用します。" + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "MLX レプリカ" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"MLX レプリカは ``model_engine=\"MLX\"`` と ``enable_xavier=True`` でプレフィックス KV " +"キャッシュを共有できます。対応モデルと FP16 の要件は :ref:`user_guide_pd_separation` " +"を参照してください。``xavier_cache_bytes`` は共有 CPU キャッシュを制限します（既定値 512 MiB）。各レプリカはモデルと" +" Metal KV " +"キャッシュも保持します。通常レプリカはミスやストレージ利用不可時にローカル計算し、互換性のないデプロイ間ではページを共有できません。使用量の " +"``prompt_tokens_details.cached_tokens`` に再利用トークン数を返します。明示的な prefill/decode " +"ロールは MLX Xavier P/D を自動で有効にします。異なるエンジン間のキャッシュ共有は未対応です。" diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po index 5bbec08c94..350484c175 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po @@ -397,3 +397,62 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "네이티브 SGLang P/D는 tokenizer worker 하나를 요구하며, 내부 HTTP 엔드포인트에는 복제본별 인증이 자동으로 적용됩니다." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "Xavier를 사용하는 MLX" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier는 Apple silicon과 ``mlx-lm>=0.31.2`` 가 필요합니다. 전체 어텐션을 사용하는 비양자화 " +"Qwen2, Qwen3, Llama 텍스트 모델을 지원하며 복제본마다 worker 하나를 사용합니다. 가중치와 KV 캐시는 FP16을 " +"사용합니다. 양자화 가중치 또는 KV 캐시, 회전 캐시, 하이브리드 어텐션, 멀티모달 입력, LoRA, 추측 디코딩은 지원하지 않습니다." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"동일한 prefill/decode 역할과 함께 ``model_engine=\"MLX\"``, " +"``model_format=\"mlx\"``, ``quantization=\"none\"`` 을 사용하세요. 예제는 두 Mac " +"worker에 복제본을 하나씩 배치합니다. 등록된 worker 주소를 ``client.get_workers_info()`` 의 결과로 " +"바꾸세요." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier는 supervisor의 공유 actor에 불변 CPU 페이지를 저장하고 actor RPC로 전송합니다. " +"``xavier_cache_bytes`` 는 배포 캐시 용량을 제한합니다(기본 512 MiB). 모델 가중치와 각 복제본의 Metal " +"KV 할당에는 추가 메모리가 필요합니다. Prefill은 마지막 프롬프트 토큰을 제외한 모든 토큰을 계산합니다. Decode는 접두사를 " +"가져와 마지막 토큰을 계산하고 첫 응답 토큰을 샘플링합니다. 스트리밍 및 비스트리밍 요청을 지원합니다. 활성 인계는 캐시 용량을 예약하며" +" 용량 부족, 메타데이터 불일치 또는 전송 실패는 오류를 발생시킵니다. 수령하지 않은 인계는 5분 후 만료됩니다." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"MLX에는 기본 ``xavier`` 전송을 사용하세요. 네이티브 NIXL은 지원하지 않습니다. 모든 worker는 supervisor의 " +"actor 주소에 접근할 수 있어야 합니다. Worker 재시작 후 배포를 다시 시작하세요. 같은 Mac의 두 복제본은 Metal " +"GPU를 공유하므로 처리량 향상을 기대하기 전에 워크로드를 측정하세요. NVIDIA에서 Mac으로의 인계를 포함한 엔진 간 P/D는 아직" +" 지원하지 않습니다." diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/vllm_enhancement.po index 30d25c7896..24f37f39a5 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -70,8 +70,8 @@ msgstr "" "``xinference-local -H 192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier는 Nvidia 그래픽 카드만 지원합니다." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "vLLM GPU 전송 경로는 NVIDIA GPU가 필요합니다." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -98,3 +98,27 @@ msgstr "일반 SGLang 복제본은 CPU HiCache 페이지를 공유합니다. 명 #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier는 BF16 체크포인트를 FP16으로 로드하고 경고를 표시합니다. 가중치와 KV 캐시 모두 FP16을 사용합니다." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "MLX 복제본" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"MLX 복제본은 ``model_engine=\"MLX\"`` 과 ``enable_xavier=True`` 로 접두사 KV 캐시를 공유할 " +"수 있습니다. 지원 모델과 FP16 요구 사항은 :ref:`user_guide_pd_separation` 에 설명되어 있습니다. " +"``xavier_cache_bytes`` 는 공유 CPU 캐시 용량을 제한합니다(기본 512 MiB). 각 복제본은 자체 모델과 " +"Metal KV 캐시도 보유합니다. 일반 복제본은 캐시 미스 또는 저장소 이용 불가 시 로컬 계산을 수행하며, 호환되지 않는 배포는 " +"페이지를 공유할 수 없습니다. 사용량의 ``prompt_tokens_details.cached_tokens`` 는 재사용된 토큰 수를 " +"보고합니다. 명시적인 prefill/decode 역할은 MLX Xavier P/D를 자동 활성화합니다. 엔진 간 캐시 공유는 아직 " +"지원하지 않습니다." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po index 7767fd8f80..b42ce64e62 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po @@ -435,3 +435,70 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "O P/D nativo do SGLang exige um único worker de tokenização; seus endpoints HTTP internos usam automaticamente autenticação por réplica." + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX com Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier requer Apple silicon e ``mlx-lm>=0.31.2``. Suporta modelos de " +"texto Qwen2, Qwen3 e Llama sem quantização, com atenção completa e um worker" +" por réplica. Os pesos e o cache KV usam FP16. Não suporta pesos ou caches " +"KV quantizados, caches rotativos, atenção híbrida, entradas multimodais, " +"LoRA ou decodificação especulativa." + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` e " +"``quantization=\"none\"`` com os mesmos papéis prefill/decode. O exemplo " +"coloca uma réplica em cada um de dois workers Mac. Substitua os endereços " +"dos workers registrados pelos retornados por ``client.get_workers_info()``." + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier armazena páginas CPU imutáveis em um actor compartilhado no " +"supervisor e as transfere por RPC de actors. ``xavier_cache_bytes`` limita " +"esse cache do deployment (padrão: 512 MiB); os pesos do modelo e as " +"alocações KV Metal de cada réplica exigem memória adicional. Prefill calcula" +" todos os tokens do prompt exceto o último. Decode importa o prefixo, " +"calcula o último token e amostra o primeiro token de resposta. Solicitações " +"com e sem streaming são suportadas. Transferências ativas reservam " +"capacidade do cache; capacidade insuficiente, metadados incompatíveis ou " +"falha de transferência geram um erro. Transferências não recebidas expiram " +"após cinco minutos." + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"Use o transporte padrão ``xavier`` para MLX; NIXL nativo não é suportado. " +"Todos os workers devem alcançar o endereço actor do supervisor. Reinicie o " +"deployment após reiniciar um worker. Duas réplicas no mesmo Mac compartilham" +" a GPU Metal; meça sua carga antes de esperar ganho de throughput. P/D entre" +" engines, incluindo transferência de NVIDIA para Mac, ainda não é suportado." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/vllm_enhancement.po index 654a571e18..8b943c529f 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -74,8 +74,8 @@ msgstr "" "exemplo: ``xinference-local -H 192.168.xx.xx``." #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier só suporta placas de vídeo Nvidia." +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "O transporte GPU do vLLM requer GPUs NVIDIA." #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -102,3 +102,30 @@ msgstr "Réplicas SGLang comuns compartilham páginas CPU HiCache. Papéis prefi #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "O Xavier carrega checkpoints BF16 como FP16 e emite um aviso. Os pesos e os caches KV usam FP16." + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "Réplicas MLX" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"Réplicas MLX podem compartilhar o cache KV de prefixos com " +"``model_engine=\"MLX\"`` e ``enable_xavier=True``. Os modelos e requisitos " +"FP16 estão descritos em :ref:`user_guide_pd_separation`. " +"``xavier_cache_bytes`` limita o cache CPU compartilhado (padrão: 512 MiB). " +"Cada réplica também possui seu modelo e cache KV Metal. Réplicas comuns " +"calculam localmente em caso de cache miss ou armazenamento indisponível; " +"deployments incompatíveis não podem compartilhar páginas. O uso informa os " +"tokens reutilizados em ``prompt_tokens_details.cached_tokens``. Papéis " +"prefill/decode explícitos ativam MLX Xavier P/D automaticamente. " +"Compartilhamento de cache entre engines ainda não é suportado." diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po index 6a8ae85f06..37c0f3cab4 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po @@ -390,3 +390,59 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "原生 SGLang P/D 要求单个 tokenizer worker；其内部 HTTP 接口自动使用每副本独立的身份认证。" + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX 与 Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier 需要 Apple 芯片和 ``mlx-lm>=0.31.2``，支持采用全注意力的未量化 Qwen2、Qwen3 和 Llama " +"文本模型，每个副本使用一个 worker。权重和 KV 缓存使用 FP16。不支持量化权重或 KV 缓存、旋转缓存、混合注意力、多模态输入、LoRA " +"和推测解码。" + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"设置 ``model_engine=\"MLX\"``、``model_format=\"mlx\"`` 和 " +"``quantization=\"none\"``，沿用相同的 prefill/decode 角色。示例在两个 Mac worker " +"上各放置一个副本。请使用 ``client.get_workers_info()`` 返回的已注册 worker 地址。" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier 在 supervisor 上的共享 actor 中保存不可变 CPU 页，通过 actor RPC " +"传输。``xavier_cache_bytes`` 限制部署缓存容量，默认 512 MiB；模型权重和各副本的 Metal KV " +"分配需要额外内存。Prefill 副本计算除最后一个之外的全部提示词 token。Decode 导入前缀，计算最后一个 token 并采样首个响应 " +"token。支持流式和非流式请求。活跃交接会预留缓存容量；容量不足、元数据不兼容或传输失败会报错。未领取的交接在五分钟后过期。" + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"MLX 使用默认的 ``xavier`` 传输，不支持原生 NIXL。所有 worker 必须能访问 supervisor 的 actor " +"地址。Worker 重启后需要重新启动部署。同一台 Mac 上的两个副本共享 Metal GPU，应先测量工作负载，再评估吞吐收益。目前不支持跨引擎 " +"P/D，包括 NVIDIA 到 Mac 的交接。" diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/vllm_enhancement.po index 9f5a8be5a8..c8cc717221 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -68,8 +68,8 @@ msgstr "" "IP 地址，例如：``xinference-local -H 192.168.xx.xx`` 。" #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier 只支持 Nvidia 显卡。" +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "vLLM GPU 传输路径需要 NVIDIA GPU。" #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -96,3 +96,26 @@ msgstr "普通 SGLang 副本共享 CPU HiCache 页面。显式 prefill/decode #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier 将 BF16 检查点加载为 FP16 并发出警告。权重和 KV 缓存均使用 FP16。" + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "MLX 副本" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"使用 ``model_engine=\"MLX\"`` 和 ``enable_xavier=True`` 可在 MLX 副本间共享前缀 KV " +"缓存。支持的模型和 FP16 要求见 :ref:`user_guide_pd_separation`。``xavier_cache_bytes`` " +"限制共享 CPU 缓存容量，默认 512 MiB。每个副本还拥有各自的模型和 Metal KV " +"缓存。普通副本在未命中或存储不可用时本地计算；不兼容的部署不能共享缓存页。响应通过 " +"``prompt_tokens_details.cached_tokens`` 报告复用 token 数。显式设置 prefill/decode " +"角色会自动启用 MLX Xavier P/D。目前不支持跨引擎缓存共享。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po index 4f67e5a06d..b1e5961168 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po @@ -388,3 +388,59 @@ msgstr "" #: ../../user_guide/pd_separation.rst msgid "Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use automatic per-replica authentication." msgstr "原生 SGLang P/D 要求單一 tokenizer worker；其內部 HTTP 介面自動使用每副本獨立的身分驗證。" + +#: user_guide/pd_separation.rst +msgid "MLX with Xavier" +msgstr "MLX 與 Xavier" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports " +"unquantized, full-attention Qwen2, Qwen3 and Llama text models with one " +"worker per replica. Weights and KV cache use FP16. Quantized weights or KV " +"cache, rotating caches, hybrid attention, multimodal inputs, LoRA and " +"speculative decoding are unsupported." +msgstr "" +"MLX Xavier 需要 Apple 晶片和 ``mlx-lm>=0.31.2``，支援採用全注意力的未量化 Qwen2、Qwen3 和 Llama " +"文字模型，每個副本使用一個 worker。權重和 KV 快取使用 FP16。不支援量化權重或 KV 快取、旋轉快取、混合注意力、多模態輸入、LoRA " +"和推測解碼。" + +#: user_guide/pd_separation.rst +msgid "" +"Use ``model_engine=\"MLX\"``, ``model_format=\"mlx\"`` and " +"``quantization=\"none\"`` with the same prefill/decode roles. The example " +"places one replica on each of two Mac workers. Replace the registered worker" +" addresses with those reported by ``client.get_workers_info()``." +msgstr "" +"設定 ``model_engine=\"MLX\"``、``model_format=\"mlx\"`` 和 " +"``quantization=\"none\"``，沿用相同的 prefill/decode 角色。範例在兩個 Mac worker " +"上各配置一個副本。請使用 ``client.get_workers_info()`` 傳回的已註冊 worker 位址。" + +#: user_guide/pd_separation.rst +msgid "" +"MLX Xavier stores immutable CPU pages in a shared actor on the supervisor " +"and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this " +"deployment cache (default: 512 MiB); model weights and per-replica Metal KV " +"allocations consume additional memory. The prefill replica evaluates all but" +" the last prompt token. Decode imports the prefix, evaluates that last token" +" and samples the first response token. Streaming and non-streaming requests " +"are supported. Active handoffs reserve cache capacity; insufficient " +"capacity, incompatible metadata or transfer failure raises an error. " +"Unclaimed handoffs expire after five minutes." +msgstr "" +"MLX Xavier 在 supervisor 的共用 actor 中儲存不可變 CPU 頁面，透過 actor RPC " +"傳輸。``xavier_cache_bytes`` 限制部署快取容量，預設 512 MiB；模型權重和各副本的 Metal KV " +"配置需要額外記憶體。Prefill 副本計算除最後一個以外的全部提示 token。Decode 匯入前綴，計算最後一個 token 並取樣首個回應 " +"token。支援串流和非串流請求。活躍交接會保留快取容量；容量不足、中繼資料不相容或傳輸失敗會報錯。未領取的交接在五分鐘後到期。" + +#: user_guide/pd_separation.rst +msgid "" +"Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " +"All workers must reach the supervisor's actor address. Relaunch after a " +"worker restart. Two replicas on one Mac share its Metal GPU; measure your " +"workload before expecting a throughput benefit. Cross-engine P/D, including " +"NVIDIA-to-Mac handoff, is not yet supported." +msgstr "" +"MLX 使用預設的 ``xavier`` 傳輸，不支援原生 NIXL。所有 worker 必須能存取 supervisor 的 actor " +"位址。Worker 重新啟動後需要重新啟動部署。同一台 Mac 上的兩個副本共用 Metal GPU，應先測量工作負載，再評估吞吐收益。目前不支援跨引擎" +" P/D，包括 NVIDIA 到 Mac 的交接。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/vllm_enhancement.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/vllm_enhancement.po index 947b6b9b1c..9d07718f0f 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/vllm_enhancement.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/vllm_enhancement.po @@ -67,8 +67,8 @@ msgstr "" "-H 192.168.xx.xx``。" #: ../../source/user_guide/vllm_enhancement.rst:23 -msgid "Xavier only works for Nvidia product." -msgstr "Xavier 只支援 Nvidia 顯示卡。" +msgid "The vLLM GPU transfer path requires NVIDIA GPUs." +msgstr "vLLM GPU 傳輸路徑需要 NVIDIA GPU。" #: source/user_guide/vllm_enhancement.rst:21 msgid "" @@ -95,3 +95,26 @@ msgstr "一般 SGLang 副本共用 CPU HiCache 頁面。明確的 prefill/decode #: ../../user_guide/vllm_enhancement.rst msgid "Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV caches use FP16." msgstr "Xavier 將 BF16 檢查點載入為 FP16 並發出警告。權重和 KV 快取均使用 FP16。" + +#: user_guide/vllm_enhancement.rst +msgid "MLX replicas" +msgstr "MLX 副本" + +#: user_guide/vllm_enhancement.rst +msgid "" +"MLX replicas can share prefix KV cache with ``model_engine=\"MLX\"`` and " +"``enable_xavier=True``. The supported models and FP16 requirements are " +"described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds " +"the shared CPU cache (default: 512 MiB). Each replica also owns its model " +"and Metal KV cache. Ordinary replicas compute locally on a miss or " +"unavailable storage; incompatible deployments cannot share pages. Returned " +"usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. " +"Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-" +"engine cache sharing is not yet supported." +msgstr "" +"使用 ``model_engine=\"MLX\"`` 和 ``enable_xavier=True`` 可在 MLX 副本間共用前綴 KV " +"快取。支援的模型和 FP16 要求見 :ref:`user_guide_pd_separation`。``xavier_cache_bytes`` " +"限制共用 CPU 快取容量，預設 512 MiB。每個副本還擁有各自的模型和 Metal KV " +"快取。普通副本在未命中或儲存不可用時於本機計算；不相容的部署不能共用快取頁面。回應透過 " +"``prompt_tokens_details.cached_tokens`` 報告重用 token 數。明確設定 prefill/decode " +"角色會自動啟用 MLX Xavier P/D。目前不支援跨引擎快取共用。" diff --git a/doc/source/user_guide/pd_separation.rst b/doc/source/user_guide/pd_separation.rst index 41fee3f836..59e44236ed 100644 --- a/doc/source/user_guide/pd_separation.rst +++ b/doc/source/user_guide/pd_separation.rst @@ -71,6 +71,36 @@ Native SGLang P/D requires one tokenizer worker; its internal HTTP endpoints use For fixed-length benchmarks, completion and chat requests accept ``ignore_eos=true``. Override model stop strings with an unused ``stop`` marker and verify the actual output token counts. +MLX with Xavier +--------------- + +MLX Xavier requires Apple silicon and ``mlx-lm>=0.31.2``. It supports unquantized, full-attention Qwen2, Qwen3 and Llama text models with one worker per replica. Weights and KV cache use FP16. Quantized weights or KV cache, rotating caches, hybrid attention, multimodal inputs, LoRA and speculative decoding are unsupported. + +Use ``model_engine="MLX"``, ``model_format="mlx"`` and ``quantization="none"`` with the same prefill/decode roles. The example places one replica on each of two Mac workers. Replace the registered worker addresses with those reported by ``client.get_workers_info()``. + +.. code-block:: python + + client.launch_model( + model_uid="qwen-mlx-pd", + model_name="qwen2.5-instruct", + model_size_in_billions="0_5", + model_engine="MLX", + model_format="mlx", + quantization="none", + replica=2, + xavier_cache_bytes=536870912, + replica_config=[ + {"role": "prefill", "devices": [{"worker_ip": "MAC_P_WORKER:PORT", + "n_gpu": "auto"}]}, + {"role": "decode", "devices": [{"worker_ip": "MAC_D_WORKER:PORT", + "n_gpu": "auto"}]}, + ], + ) + +MLX Xavier stores immutable CPU pages in a shared actor on the supervisor and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this deployment cache (default: 512 MiB); model weights and per-replica Metal KV allocations consume additional memory. The prefill replica evaluates all but the last prompt token. Decode imports the prefix, evaluates that last token and samples the first response token. Streaming and non-streaming requests are supported. Active handoffs reserve cache capacity; insufficient capacity, incompatible metadata or transfer failure raises an error. Unclaimed handoffs expire after five minutes. + +Use the default ``xavier`` transport for MLX; native NIXL is unsupported. All workers must reach the supervisor's actor address. Relaunch after a worker restart. Two replicas on one Mac share its Metal GPU; measure your workload before expecting a throughput benefit. Cross-engine P/D, including NVIDIA-to-Mac handoff, is not yet supported. + Native NIXL backend ------------------- diff --git a/doc/source/user_guide/vllm_enhancement.rst b/doc/source/user_guide/vllm_enhancement.rst index 696494b9a3..3b26a529ae 100644 --- a/doc/source/user_guide/vllm_enhancement.rst +++ b/doc/source/user_guide/vllm_enhancement.rst @@ -22,7 +22,7 @@ Limitations The V1 connector supports vLLM >= ``0.21.0``. See :ref:`user_guide_pd_separation` for community Prefill/Decode deployment. * Due to the underlying communication not recognizing ``0.0.0.0``, the actual IP address needs to be passed when starting Xinference, for example: ``xinference-local -H 192.168.xx.xx``. -* Xavier only works for Nvidia product. +* The vLLM GPU transfer path requires NVIDIA GPUs. SGLang replicas =============== @@ -34,3 +34,8 @@ Xavier loads BF16 checkpoints as FP16 and emits a warning. Both weights and KV c ``xavier_cache_bytes`` sets the deployment's shared CPU cache budget (default: ``536870912``, 512 MiB). Each SGLang replica also allocates its own GPU and HiCache host pools. Eviction or unavailable storage becomes a cache miss for ordinary replicas. Cache namespaces include the actual weights, tokenizer, model configuration, KV geometry and engine format; caches cannot yet be exchanged between vLLM and SGLang. Ordinary SGLang replicas share CPU HiCache pages. Explicit prefill/decode roles use Xavier GPU-to-GPU transfer instead; see :ref:`user_guide_pd_separation`. Cross-engine P/D is not yet supported. + +MLX replicas +============ + +MLX replicas can share prefix KV cache with ``model_engine="MLX"`` and ``enable_xavier=True``. The supported models and FP16 requirements are described in :ref:`user_guide_pd_separation`. ``xavier_cache_bytes`` bounds the shared CPU cache (default: 512 MiB). Each replica also owns its model and Metal KV cache. Ordinary replicas compute locally on a miss or unavailable storage; incompatible deployments cannot share pages. Returned usage reports reused tokens in ``prompt_tokens_details.cached_tokens``. Explicit prefill/decode roles enable MLX Xavier P/D automatically. Cross-engine cache sharing is not yet supported. diff --git a/frontend/src/components/pages/launch-model/launch-dialog/replica-placement-config.tsx b/frontend/src/components/pages/launch-model/launch-dialog/replica-placement-config.tsx index 60309d2ccb..89a6b0615d 100644 --- a/frontend/src/components/pages/launch-model/launch-dialog/replica-placement-config.tsx +++ b/frontend/src/components/pages/launch-model/launch-dialog/replica-placement-config.tsx @@ -40,7 +40,7 @@ const ReplicaPlacementConfig: FC = ({ form, workerO const { t } = useI18n(); const { notifyUserChange } = useFormContext(); const engine = useWatch('model_engine', form); - const supportsPD = ['vllm', 'sglang'].includes(String(engine).toLowerCase()); + const supportsPD = ['vllm', 'sglang', 'mlx'].includes(String(engine).toLowerCase()); const columns = supportsPD ? 'grid-cols-[1fr_2fr_1fr_1fr]' : 'grid-cols-[1fr_2fr_1fr]'; const rows = (useWatch('replica_config', form) as ReplicaConfigRow[] | undefined) ?? []; diff --git a/xinference/core/pd_model.py b/xinference/core/pd_model.py index db003714df..73672ec09f 100644 --- a/xinference/core/pd_model.py +++ b/xinference/core/pd_model.py @@ -257,7 +257,7 @@ async def free_prefill_model_cache(self, request_id: str): handoff = self._direct_transfers.pop(request_id, None) if handoff: try: - if handoff.get("engine") == "sglang": + if handoff.get("engine") in ("sglang", "mlx"): ref = await xo.actor_ref( address=handoff["address"], uid=handoff["uid"] ) @@ -359,7 +359,11 @@ async def _infer(self, method, inputs, *args, **kwargs): else None ) if self._direct_handoff and isinstance(transfer, dict): - handoff = transfer.get("xavier_direct") or transfer.get("sglang_xavier") + handoff = ( + transfer.get("xavier_direct") + or transfer.get("sglang_xavier") + or transfer.get("mlx_xavier") + ) if handoff: self._direct_transfers[request_id] = handoff if not isinstance(transfer, dict) or not transfer.get("do_remote_prefill"): diff --git a/xinference/core/replica_config.py b/xinference/core/replica_config.py index c38869817d..ff51aa86b1 100644 --- a/xinference/core/replica_config.py +++ b/xinference/core/replica_config.py @@ -183,11 +183,11 @@ def validate_pd_replica_configs( roles = {cfg.role for cfg in configs or []} if not roles.intersection({"prefill", "decode"}): return False - if (model_engine or "").lower() not in ("vllm", "sglang") or ( + if (model_engine or "").lower() not in ("vllm", "sglang", "mlx") or ( model_type or "LLM" ).lower() != "llm": raise ValueError( - "PD separation requires model_type=LLM and model_engine=vLLM or SGLang" + "PD separation requires model_type=LLM and model_engine=vLLM, SGLang or MLX" ) if roles != {"prefill", "decode"}: raise ValueError( diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index 6a92cfbc49..50721d06bd 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3112,6 +3112,26 @@ async def launch_builtin_model( raise ValueError("NIXL requires explicit prefill and decode replica roles") # Xavier-related requested_xavier = bool(kwargs.pop("enable_xavier", False)) + mlx_xavier = ( + (requested_xavier or pd_enabled) + and transport_backend == "xavier" + and (model_engine or "").lower() == "mlx" + ) + if (requested_xavier or pd_enabled) and (model_engine or "").lower() == "mlx": + if transport_backend != "xavier" or n_worker != 1: + raise ValueError( + "MLX Xavier requires the xavier transport and one worker per replica" + ) + if ( + model_type not in (None, "LLM") + or model_format != "mlx" + or quantization not in (None, "none") + ): + raise ValueError("MLX Xavier requires unquantized MLX text weights") + kwargs["_xavier_cache_config"] = { + "address": self.address, + "uid": f"xavier-cache-{model_uid}", + } sglang_xavier = ( (requested_xavier or pd_enabled) and transport_backend == "xavier" @@ -3133,13 +3153,12 @@ async def launch_builtin_model( ) cache_bytes = kwargs.pop("xavier_cache_bytes", None) if cache_bytes is not None and ( - not sglang_xavier - or pd_enabled + not (mlx_xavier or sglang_xavier and not pd_enabled) or type(cache_bytes) is not int or cache_bytes <= 0 ): raise ValueError( - "xavier_cache_bytes requires SGLang shared CPU caching and a positive integer" + "xavier_cache_bytes requires MLX Xavier or SGLang shared CPU caching and a positive integer" ) if sglang_xavier: if n_worker != 1: @@ -3290,6 +3309,12 @@ async def _launch_one_model( "rank": rank, "host": get_transport_host(worker_ref.address), } + elif mlx_xavier and pd_enabled: + assert replica_config is not None + replica_kwargs["_xavier_cache_config"] = { + **kwargs["_xavier_cache_config"], + "role": replica_config[rank - 1].role, + } if pd_enabled and transport_backend == "nixl": assert replica_config is not None replica_kwargs["_nixl_config"] = { @@ -3352,8 +3377,19 @@ async def _launch_one_model( async def _launch_model(): nonlocal download_hub try: - if sglang_xavier: - if pd_enabled: + if sglang_xavier or mlx_xavier: + if mlx_xavier: + from ..model.llm.xavier.backends.bytes.storage import ( + XavierBytesCacheActor, + ) + + cache_ref = await xo.create_actor( + XavierBytesCacheActor, + capacity_bytes=cache_bytes or 512 * 1024 * 1024, + address=self.address, + uid=kwargs["_xavier_cache_config"]["uid"], + ) + elif pd_enabled: from ..model.llm.sglang.xavier.directory import ( XavierPDDirectory, ) diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index d6bb1eecf9..32aa00f216 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -67,6 +67,76 @@ def launch_kwargs(): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("pd", [False, True]) +async def test_mlx_xavier_launch_and_cleanup(launch_runtime, pd): + supervisor, workers, actors, destroy = launch_runtime + kwargs = launch_kwargs() + kwargs.update(model_engine="MLX", model_format="mlx", xavier_cache_bytes=123456) + if not pd: + kwargs["enable_xavier"] = True + for replica in kwargs["replica_config"]: + replica.role = "hybrid" + await supervisor.launch_builtin_model(**kwargs) + assert set(actors) == ( + {"XavierBytesCacheActor", "PDModelActor"} if pd else {"XavierBytesCacheActor"} + ) + assert ( + actors["XavierBytesCacheActor"].constructor_kwargs["capacity_bytes"] == 123456 + ) + for role, worker in zip(("prefill", "decode"), workers): + launch = worker.launch_builtin_model.call_args.kwargs + config = {"address": supervisor.address, "uid": "xavier-cache-pd"} + if pd: + config["role"] = role + assert launch["_xavier_cache_config"] == config + assert launch["xavier_config"] is None + assert "_nixl_config" not in launch + worker.launch_rank0_model.assert_not_awaited() + worker.start_transfer_for_vllm.assert_not_awaited() + await supervisor.terminate_model("pd") + assert not supervisor._xavier_cache_mapping and not supervisor._pd_model_mapping + assert destroy.await_count == (2 if pd else 1) + + +@pytest.mark.asyncio +async def test_mlx_failed_launch_cleans_bytes_actor(launch_runtime): + supervisor, workers, actors, destroy = launch_runtime + kwargs = launch_kwargs() + kwargs.update(model_engine="MLX", model_format="mlx") + workers[1].wait_for_load.side_effect = RuntimeError("bad model") + with pytest.raises(RuntimeError, match="bad model"): + await supervisor.launch_builtin_model(**kwargs) + assert not supervisor._xavier_cache_mapping + assert not supervisor._model_uid_to_replica_info + destroy.assert_awaited_once_with(actors["XavierBytesCacheActor"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "options", + [ + {"transfer_backend_type": "nixl"}, + {"n_worker": 2}, + {"model_format": "pytorch"}, + {"quantization": "4bit"}, + {"xavier_gpu_cache_bytes": 1024}, + {"xavier_cache_bytes": 0}, + {"xavier_cache_bytes": True}, + ], +) +async def test_mlx_invalid_launch_has_no_actors(launch_runtime, options): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + kwargs.update(model_engine="MLX", model_format="mlx") + kwargs.update(options) + with pytest.raises(ValueError): + await supervisor.launch_builtin_model(**kwargs) + assert not actors and not supervisor._model_uid_to_replica_info + for worker in workers: + worker.launch_builtin_model.assert_not_awaited() + + @pytest.mark.asyncio async def test_sglang_xavier_uses_hicache_without_vllm_collective(launch_runtime): supervisor, workers, actors, destroy = launch_runtime diff --git a/xinference/core/tests/test_pd_model.py b/xinference/core/tests/test_pd_model.py index 21adace1a0..a015b17231 100644 --- a/xinference/core/tests/test_pd_model.py +++ b/xinference/core/tests/test_pd_model.py @@ -28,6 +28,7 @@ def test_round_robin_updates(): (["prefill", "decode"], "vLLM", True), (["prefill", "prefill", "decode"], "vLLM", True), (["prefill", "decode"], "SGLang", True), + (["prefill", "decode"], "MLX", True), (["hybrid"], "transformers", False), (["prefill"], "vLLM", None), (["decode"], "vLLM", None), @@ -458,7 +459,7 @@ async def test_duplicate_generation_config_rejected_before_dispatch(router): @pytest.mark.asyncio @pytest.mark.parametrize("failure", ["decode", "cancel", "abort_before_decode"]) -@pytest.mark.parametrize("engine", ["vllm", "sglang"]) +@pytest.mark.parametrize("engine", ["vllm", "sglang", "mlx"]) async def test_direct_router_releases_unclaimed_handoff( router, monkeypatch, failure, engine ): @@ -468,11 +469,11 @@ async def test_direct_router_releases_unclaimed_handoff( "do_remote_prefill": True, "xavier_direct": {"ticket": "t", "rank": 0, "address": "127.0.0.1:1234"}, } - if engine == "sglang": + if engine in ("sglang", "mlx"): transfer = { "do_remote_prefill": True, - "sglang_xavier": { - "engine": "sglang", + f"{engine}_xavier": { + "engine": engine, "ticket": "t", "uid": "cache", "address": "127.0.0.1:1234", @@ -494,7 +495,7 @@ async def finish_prefill(*args, **kwargs): monkeypatch.setattr(xo, "actor_ref", AsyncMock(return_value=peer)) with pytest.raises(RuntimeError if failure == "decode" else asyncio.CancelledError): await actor._infer("chat", [], {}, request_id="r") - if engine == "sglang": + if engine in ("sglang", "mlx"): peer.release_handoff.assert_awaited_once_with("t") peer.abandon_direct_gpu_v1.assert_not_awaited() else: diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index 73218cbaab..e01f698583 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -37,6 +37,7 @@ Tuple, TypedDict, Union, + cast, ) import xoscar as xo @@ -121,6 +122,7 @@ def __init__( max_context_length: int = 2048, prompt_cache_size: int = _DEFAULT_MLX_PROMPT_CACHE_SIZE, prompt_cache_bytes: Optional[int] = None, + xavier=None, ): # Store references for creating new generators on demand MLXBatchModel._model_ref = model @@ -133,6 +135,7 @@ def __init__( MLXBatchModel._stop_tokens = set(eos_token_ids or []) self._prompt_cache: Optional[Any] = None self._prompt_cache_model_key = id(model) + self._xavier = xavier if self._is_new_mlx_lm() and prompt_cache_size > 0: try: from mlx_lm.models.cache import LRUPromptCache @@ -222,6 +225,7 @@ def _get_or_create_generator( "pending": {}, # uid -> list of results "active": set(), # active uids "cache_boundaries": {}, # uid -> stable prefix token count + "xavier_prompts": {}, "task": None, } @@ -261,14 +265,19 @@ def _insert_request( prompt_tokens: List[int], max_tokens: int, prompt_cache_prefix_len: Optional[int], + external_cache=None, ) -> Tuple[int, int, Optional[int]]: if not self._is_new_mlx_lm(): request_ids = batch_generator.insert([prompt_tokens], max_tokens=max_tokens) return request_ids[0], 0, None - cache, remaining_tokens, cached_token_count = self._fetch_prompt_cache( - prompt_tokens - ) + if external_cache is not None and external_cache[0] is not None: + cache, cached_token_count = external_cache + remaining_tokens = prompt_tokens[cached_token_count:] + else: + cache, remaining_tokens, cached_token_count = self._fetch_prompt_cache( + prompt_tokens + ) insert_kwargs: Dict[str, Any] = {"max_tokens": [max_tokens]} if cache is not None: insert_kwargs.update( @@ -329,6 +338,19 @@ def _ensure_background_worker(self, gen_dict): loop = asyncio.get_event_loop() gen_dict["task"] = loop.create_task(self._background_worker(gen_dict)) + def _store_xavier_prompt_caches(self, batch_generator, prompt_results, gen_dict): + if getattr(self, "_xavier", None) is None: + return + prompts = gen_dict["xavier_prompts"] + for result in prompt_results: + if not result.end_of_prompt or result.uid not in prompts: + continue + extracted = batch_generator.extract_cache([result.uid]).get(result.uid) + if extracted is not None: + cache, _ = extracted + tokens = prompts.pop(result.uid) + self._xavier.publish(cache, tokens[:-1]) + async def _background_worker(self, gen_dict): """Background worker that continuously calls next() and distributes results.""" key = f"temp={gen_dict['generator'].sampler}" @@ -345,6 +367,9 @@ async def _background_worker(self, gen_dict): self._store_segment_prompt_caches( batch_generator, prompt_results, gen_dict ) + self._store_xavier_prompt_caches( + batch_generator, prompt_results, gen_dict + ) else: # Old API (mlx-lm < 0.31.2): use next() and unpack batch_results = batch_generator.next() @@ -400,6 +425,9 @@ async def generate_stream( request_id: Optional[str] = None, skip_special_tokens: bool = True, prompt_cache_prefix_len: Optional[int] = None, + kv_transfer_params: Optional[dict] = None, + prepared_cache=None, + prompt_token_ids: Optional[List[int]] = None, ) -> AsyncGenerator[CompletionChunk, None]: """Async stream generate method using BatchGenerator.""" # Get or create generator for this sampling configuration @@ -411,18 +439,35 @@ async def generate_stream( # Prepare request assert MLXBatchModel._tokenizer_ref is not None - prompt_tokens = MLXBatchModel._tokenizer_ref.encode(prompt) + prompt_tokens = ( + prompt_token_ids + if prompt_token_ids is not None + else MLXBatchModel._tokenizer_ref.encode(prompt) + ) input_echo_len = len(prompt_tokens) # Create queue first queue: asyncio.Queue = asyncio.Queue() - inserted_uid, cached_prompt_tokens, cache_boundary = self._insert_request( + external_cache = prepared_cache + if external_cache is None and getattr(self, "_xavier", None) is not None: + external_cache = await self._xavier.fetch(prompt_tokens, kv_transfer_params) + insert_args = ( batch_generator, prompt_tokens, max_tokens, prompt_cache_prefix_len, ) + inserted_uid, cached_prompt_tokens, cache_boundary = ( + self._insert_request(*insert_args, external_cache=external_cache) + if external_cache is not None + else self._insert_request(*insert_args) + ) + if ( + getattr(self, "_xavier", None) is not None + and cached_prompt_tokens < input_echo_len - 1 + ): + gen_dict["xavier_prompts"][inserted_uid] = prompt_tokens if cache_boundary is not None: gen_dict["cache_boundaries"][inserted_uid] = cache_boundary @@ -557,6 +602,11 @@ async def generate_stream( if inserted_uid in gen_dict["active"]: gen_dict["active"].remove(inserted_uid) gen_dict["cache_boundaries"].pop(inserted_uid, None) + if getattr(self, "_xavier", None) is not None: + gen_dict["xavier_prompts"].pop(inserted_uid, None) + batch_generator.remove([inserted_uid]) + if getattr(self, "_xavier", None) is not None: + await self._xavier.flush() async def generate( self, @@ -568,6 +618,9 @@ async def generate( stream: bool = False, skip_special_tokens: bool = True, prompt_cache_prefix_len: Optional[int] = None, + kv_transfer_params: Optional[dict] = None, + prepared_cache=None, + prompt_token_ids: Optional[List[int]] = None, ) -> Tuple[str, CompletionUsage]: """Async generate method using BatchGenerator.""" # Non-streaming: collect all tokens @@ -587,6 +640,9 @@ async def generate( request_id=None, skip_special_tokens=skip_special_tokens, prompt_cache_prefix_len=prompt_cache_prefix_len, + kv_transfer_params=kv_transfer_params, + prepared_cache=prepared_cache, + prompt_token_ids=prompt_token_ids, ): if chunk.get("usage") is not None: usage = chunk["usage"] @@ -596,6 +652,7 @@ async def generate( class MLXModelConfig(TypedDict, total=False): + _xavier_cache_config: dict revision: Optional[str] max_gpu_memory: str trust_remote_code: bool @@ -618,6 +675,7 @@ class MLXModelConfig(TypedDict, total=False): class MLXGenerateConfig(TypedDict, total=False): + _pd_kv_transfer_params: dict max_tokens: int temperature: float repetition_penalty: Optional[float] @@ -676,6 +734,8 @@ def __init__( # used to call async self._loop = None self._batch_model = None + self._xavier_config = self._model_config.pop("_xavier_cache_config", None) + self._xavier = None def set_loop(self, loop: asyncio.AbstractEventLoop): # loop will be passed into ModelWrapper, @@ -1045,6 +1105,20 @@ def wait_for_load(self): config.update(self._model_config) self._update_model_generation_config(config) self._context_length = get_context_length_from_config(config) + if self._xavier_config is not None: + from .xavier import configure_xavier + + self._xavier = configure_xavier( + self._model, + self.model_path, + self._model_config, + self._xavier_config, + self._n_worker, + ) + if not self.allow_batch: + raise ValueError( + "MLX Xavier requires the continuous batching text engine" + ) # Update allow_batch based on distributed inference # Only enable continuous batching for non-distributed inference (single worker) @@ -1073,6 +1147,7 @@ def wait_for_load(self): max_context_length=self._context_length, prompt_cache_size=prompt_cache_size, prompt_cache_bytes=prompt_cache_bytes, + xavier=self._xavier, ) @classmethod @@ -1333,6 +1408,60 @@ async def async_generate( assert self._model is not None assert self._tokenizer is not None + transfer = generate_config.get("_pd_kv_transfer_params") + xavier = getattr(self, "_xavier", None) + xavier_inputs = {} + if xavier is not None: + if xavier.role == "prefill": + if ( + not isinstance(transfer, dict) + or not transfer.get("do_remote_decode") + or generate_config.get("stream") + ): + raise ValueError( + "MLX prefill replicas require a non-streaming PD request" + ) + if not isinstance(prompt, str): + raise ValueError("MLX Xavier requires a text prompt") + tokens = self._tokenizer.encode(prompt) + metadata = await xavier.prefill( + self._model, tokens, generate_config.get("prompt_cache_prefix_len") + ) + return Completion( + id=str(uuid.uuid4()), + object="text_completion", + created=int(time.time()), + model=self.model_uid, + choices=[ + dict(text="", index=0, logprobs=None, finish_reason="stop") + ], + usage=CompletionUsage( + prompt_tokens=len(tokens), + completion_tokens=0, + total_tokens=len(tokens), + ), + _pd_kv_transfer_params=metadata, + ) + if xavier.role == "decode" and ( + not isinstance(transfer, dict) or not transfer.get("do_remote_prefill") + ): + raise ValueError( + "MLX decode replicas require Xavier PD handoff metadata" + ) + if xavier.role is None and transfer is not None: + raise ValueError( + "MLX remote prefill requires explicit P/D replica roles" + ) + if not isinstance(prompt, str): + raise ValueError("MLX Xavier requires a text prompt") + tokens = self._tokenizer.encode(prompt) + # Claim/copy/release the lease before returning a lazy stream. Even + # an unconsumed or disconnected response cannot strand the handoff. + xavier_inputs = dict( + prepared_cache=await xavier.fetch(tokens, transfer), + prompt_token_ids=tokens, + ) + stream = generate_config.get("stream", False) # If batch model is available (non-distributed inference), use continuous batching @@ -1370,6 +1499,7 @@ async def stream_generator(): prompt_cache_prefix_len=generate_config.get( "prompt_cache_prefix_len" ), + **xavier_inputs, ): # Set model_uid for each chunk if hasattr(chunk, "get"): @@ -1394,6 +1524,7 @@ async def stream_generator(): prompt_cache_prefix_len=generate_config.get( "prompt_cache_prefix_len" ), + **xavier_inputs, ) # Return completion object @@ -1602,6 +1733,11 @@ async def _log_streaming_chunks(): assert not hasattr( c, "__aiter__" ), "async_generate should return Completion for non-streaming" + c = cast(Completion, c) + if c.get("_pd_kv_transfer_params"): + result = self._to_chat_completion(c, self.reasoning_parser) + result["_pd_kv_transfer_params"] = c["_pd_kv_transfer_params"] + return result if tools: return self._post_process_completion( self.model_family, self.model_uid, c diff --git a/xinference/model/llm/mlx/tests/test_xavier.py b/xinference/model/llm/mlx/tests/test_xavier.py new file mode 100644 index 0000000000..d3e824c784 --- /dev/null +++ b/xinference/model/llm/mlx/tests/test_xavier.py @@ -0,0 +1,312 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +import asyncio +import importlib +import json +import platform +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +import xoscar as xo + +from ...xavier.backends.bytes.storage import XavierBytesCacheActor +from ...xavier.contract import KVCacheContract +from ..xavier import MLXXavierCache, configure_xavier + +metal = pytest.mark.skipif( + platform.system() != "Darwin" or platform.machine() != "arm64", + reason="Requires Apple Metal", +) + + +def contract(): + return KVCacheContract( + *["0" * 64] * 4, + num_layers=2, + num_kv_heads=2, + head_dim=8, + block_size=64, + logical_dtype="float16", + ) + + +def model(model_type="qwen2"): + import mlx.core as mx + + module = importlib.import_module(f"mlx_lm.models.{model_type}") + mx.random.seed(42) + args = dict( + model_type=model_type, + hidden_size=32, + num_hidden_layers=2, + intermediate_size=64, + num_attention_heads=4, + rms_norm_eps=1e-6, + vocab_size=128, + num_key_value_heads=2, + ) + if model_type == "qwen3": + args.update( + head_dim=8, + max_position_embeddings=2048, + rope_theta=10000.0, + tie_word_embeddings=False, + ) + value = module.Model(module.ModelArgs(**args)) + value.set_dtype(mx.float16) + mx.eval(value.parameters()) + return value + + +@pytest.mark.asyncio +async def test_pd_metadata_is_checked_before_reading(): + client = MLXXavierCache( + contract(), dict(address="address", uid="cache", role="decode") + ) + client._call = AsyncMock() + with pytest.raises(ValueError, match="handoff"): + await client.fetch( + [1, 2, 3], {"mlx_xavier": dict(engine="mlx", address="wrong")} + ) + client._call.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_decode_claims_before_returning_lazy_stream(): + from ..core import MLXModel + + wrapper = object.__new__(MLXModel) + wrapper._model = object() + wrapper._tokenizer = SimpleNamespace(encode=lambda prompt: [1, 2, 3]) + wrapper._xavier = SimpleNamespace( + role="decode", fetch=AsyncMock(return_value=([], 2)) + ) + wrapper._model_generation_config = {} + wrapper.model_uid = "mlx" + wrapper._batch_model = SimpleNamespace(generate_stream=AsyncMock()) + transfer = {"do_remote_prefill": True} + stream = await wrapper.async_generate( + "prompt", {"stream": True, "max_tokens": 2, "_pd_kv_transfer_params": transfer} + ) + wrapper._xavier.fetch.assert_awaited_once_with([1, 2, 3], transfer) + wrapper._batch_model.generate_stream.assert_not_called() + await stream.aclose() + + +@metal +@pytest.mark.parametrize( + "config", + [ + {"model_type": "qwen3_5"}, + {"model_type": "qwen2", "quantization": {"bits": 4}}, + {"model_type": "qwen2", "use_sliding_window": True}, + {"model_type": "qwen2", "layer_types": ["linear_attention"]}, + ], +) +def test_unsupported_model_contract_rejected(tmp_path, config): + (tmp_path / "config.json").write_text(json.dumps(config)) + with pytest.raises(ValueError, match="full-attention"): + configure_xavier(model(), tmp_path, {}, {}, 1) + + +@metal +def test_effective_model_overrides_change_namespace(tmp_path): + (tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen2"})) + (tmp_path / "model.safetensors").write_bytes(b"fingerprint-fixture") + (tmp_path / "tokenizer.json").write_text("{}") + m = model() + first = configure_xavier(m, tmp_path, {}, {}, 1).contract + m.args.rope_theta *= 2 + second = configure_xavier(m, tmp_path, {}, {}, 1).contract + with pytest.raises(ValueError, match="fingerprint"): + first.require_match(second) + + +@metal +@pytest.mark.parametrize("model_type", ["qwen2", "qwen3", "llama"]) +def test_supported_architectures_use_actual_kv_geometry(tmp_path, model_type): + import mlx.core as mx + from mlx_lm.models.cache import make_prompt_cache + + (tmp_path / "config.json").write_text(json.dumps({"model_type": model_type})) + (tmp_path / "model.safetensors").write_bytes(b"fingerprint-fixture") + (tmp_path / "tokenizer.json").write_text("{}") + m = model(model_type) + client = configure_xavier(m, tmp_path, {}, {}, 1) + cache = make_prompt_cache(m) + tokens = [1, 2, 3] + mx.eval(m(mx.array(tokens)[None], cache=cache)) + restored = client.decode(client.encode(cache, tokens), len(tokens)) + assert client.contract.head_dim == 8 + assert all( + bool(mx.array_equal(a, b)) + for old, new in zip(cache, restored) + for a, b in zip(old.state, new.state) + ) + + +@metal +@pytest.mark.parametrize("length", [0, 1, 63, 64, 65, 129]) +def test_real_metal_canonical_roundtrip(length): + import mlx.core as mx + from mlx_lm.models.cache import make_prompt_cache + + m = model() + cache = make_prompt_cache(m) + tokens = [i % 128 for i in range(length)] + if length: + mx.eval(m(mx.array(tokens)[None], cache=cache)) + client = MLXXavierCache(contract(), {}) + pages = client.encode(cache, tokens) + restored = client.decode(pages, length) + assert [c.offset for c in restored] == [length] * 2 + if length: + assert all( + bool(mx.array_equal(a, b)) + for old, new in zip(cache, restored) + for a, b in zip(old.state, new.state) + ) + cache[0].keys = cache[0].keys.astype(mx.bfloat16) + with pytest.raises(ValueError, match="dtype"): + client.encode(cache, tokens) + + +@metal +@pytest.mark.asyncio +async def test_real_pd_import_skips_prefill_and_matches_logits(): + import mlx.core as mx + from mlx_lm.generate import BatchGenerator + from mlx_lm.models.cache import make_prompt_cache + + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="mlx-cache" + ) + cfg = dict(address=ref.address, uid="mlx-cache") + p = MLXXavierCache(contract(), dict(cfg, role="prefill")) + d = MLXXavierCache(contract(), dict(cfg, role="decode")) + tokens = [i % 128 for i in range(130)] + m = model() + transfer = await p.prefill(m, tokens, prefix_length=24) + cache, reused = await d.fetch(tokens, transfer) + assert reused == len(tokens) - 1 and d.imported_tokens == reused + reference = BatchGenerator(m) + uid = reference.insert_segments([[tokens[:24], tokens[24:]]], max_tokens=[1])[0] + while True: + prompts, _ = reference.next() + if any(r.end_of_prompt for r in prompts): + break + native_cache, _ = reference.extract_cache([uid])[uid] + assert all( + bool(mx.array_equal(a[:, :, :reused], b)) + for native, restored in zip(native_cache, cache) + for a, b in zip(native.state, restored.state) + ) + # D evaluates exactly one prompt token against imported cache state. + actual = m(mx.array(tokens[reused:])[None], cache=cache)[:, -1] + expected = m(mx.array(tokens)[None], cache=make_prompt_cache(m))[:, -1] + mx.eval(actual, expected) + assert bool(mx.array_equal(mx.argmax(actual, -1), mx.argmax(expected, -1))) + assert ( + float( + mx.max(mx.abs(actual.astype(mx.float32) - expected.astype(mx.float32))) + ) + < 0.02 + ) + stats = await ref.get_stats() + assert ( + stats["handoff_reads"] == 1 + and stats["read_pages"] == 3 + and stats["active_handoffs"] == 0 + ) + with pytest.raises(ValueError, match="handoff"): + await d.fetch(tokens, transfer) + p.imported_tokens = 0 + next_transfer = await p.prefill(m, tokens, prefix_length=24) + assert p.imported_tokens == 0 # Complete warm pages stay on CPU until D reads. + await d.fetch(tokens, next_transfer) + assert (await ref.get_stats())["active_handoffs"] == 0 + + +@metal +@pytest.mark.asyncio +async def test_pd_failed_prefill_releases_reserved_pages(): + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="mlx-cache" + ) + p = MLXXavierCache( + contract(), dict(address=ref.address, uid="mlx-cache", role="prefill") + ) + m = model() + + def fail(*args, **kwargs): + raise RuntimeError("prefill failed") + + m.__class__.__call__, saved = fail, m.__class__.__call__ + try: + with pytest.raises(RuntimeError, match="prefill failed"): + await p.prefill(m, [1, 2, 3]) + finally: + m.__class__.__call__ = saved + assert (await ref.get_stats())["active_handoffs"] == 0 + + +@metal +@pytest.mark.asyncio +async def test_real_batch_generation_reuses_remote_cache(monkeypatch): + from ..core import MLXBatchModel + + class Tokenizer: + eos_token_ids = [] + + def encode(self, text): + return list(text.encode()) + + def decode(self, tokens, **kwargs): + return "".join(f"<{t}>" for t in tokens) + + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="mlx-cache" + ) + cfg = dict(address=ref.address, uid="mlx-cache") + clients = [] + + async def generate(role=None, transfer=None): + monkeypatch.setattr(MLXBatchModel, "_batch_generators", {}) + monkeypatch.setattr(MLXBatchModel, "_lock", None) + client = MLXXavierCache(contract(), dict(cfg, role=role)) + client.publish = MagicMock(wraps=client.publish) + clients.append(client) + wrapper = MLXBatchModel( + model(), Tokenizer(), prompt_cache_size=0, xavier=client + ) + try: + return await wrapper.generate( + "a" * 70, 8, temperature=0, kv_transfer_params=transfer + ) + finally: + tasks = [ + g["task"] for g in wrapper._batch_generators.values() if g["task"] + ] + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + cold, cold_usage = await generate() + warm, warm_usage = await generate() + assert ( + warm == cold and cold_usage["prompt_tokens_details"]["cached_tokens"] == 0 + ) + assert warm_usage["prompt_tokens_details"]["cached_tokens"] == 69 + p = MLXXavierCache(contract(), dict(cfg, role="prefill")) + transfer = await p.prefill(model(), Tokenizer().encode("a" * 70)) + pd, pd_usage = await generate("decode", transfer) + assert pd == cold and pd_usage["prompt_tokens_details"]["cached_tokens"] == 69 + assert [c.publish.call_count for c in clients] == [1, 0, 0] + assert (await ref.get_stats())["handoff_reads"] == 1 diff --git a/xinference/model/llm/mlx/xavier.py b/xinference/model/llm/mlx/xavier.py new file mode 100644 index 0000000000..b3c556beff --- /dev/null +++ b/xinference/model/llm/mlx/xavier.py @@ -0,0 +1,334 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""Canonical Xavier pages for full-attention MLX caches and explicit P/D.""" + +import asyncio +import importlib.metadata +import json +import logging +from dataclasses import asdict +from pathlib import Path + +import numpy as np +import xoscar as xo + +from ..xavier.contract import ( + KVCacheContract, + build_prefix_keys, + fingerprint_files, + fingerprint_metadata, +) + +logger = logging.getLogger(__name__) + + +def configure_xavier(model, model_path, model_config, cache_config, n_worker): + import mlx.core as mx + from mlx_lm.models.cache import KVCache, make_prompt_cache + from packaging.version import Version + + if ( + n_worker != 1 + or model_config.get("max_kv_size") + or model_config.get("draft_model_path") + or Version(importlib.metadata.version("mlx-lm")) < Version("0.31.2") + ): + raise ValueError( + "MLX Xavier requires mlx-lm>=0.31.2, one worker and full unquantized KV caches" + ) + path = Path(model_path) + config = json.loads((path / "config.json").read_text()) + if ( + config.get("model_type") not in ("qwen2", "qwen3", "llama") + or any( + config.get(k) + for k in ( + "quantization", + "quantization_config", + "vision_config", + "auto_map", + "use_sliding_window", + ) + ) + or any(t != "full_attention" for t in config.get("layer_types", [])) + ): + raise ValueError( + "MLX Xavier supports unquantized full-attention Qwen2, Qwen3 and Llama text models" + ) + if not all(type(c) is KVCache for c in make_prompt_cache(model)): + raise ValueError("MLX Xavier requires ordinary full-attention KVCache layers") + tokenizer_files = { + p.name: p + for p in path.iterdir() + if p.name + in ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "tokenizer.model", + "spiece.model", + ) + and p.is_file() + } + if not tokenizer_files or not any( + n in tokenizer_files + for n in ("tokenizer.json", "tokenizer.model", "spiece.model", "vocab.json") + ): + raise ValueError("MLX Xavier requires local tokenizer assets") + tokenizer_config = ( + json.loads(tokenizer_files["tokenizer_config.json"].read_text()) + if "tokenizer_config.json" in tokenizer_files + else {} + ) + if tokenizer_config.get("auto_map"): + raise ValueError("MLX Xavier requires a standard tokenizer") + weights = {p.name: p for p in path.glob("*.safetensors")} + if not weights: + raise ValueError("MLX Xavier requires local safetensors weights") + model.set_dtype(mx.float16) + args = asdict(model.args) + contract = KVCacheContract( + weights_fingerprint=fingerprint_files(weights), + tokenizer_fingerprint=fingerprint_files(tokenizer_files), + attention_fingerprint=fingerprint_metadata( + {"config": config, "effective_args": args, "dtype": "float16"} + ), + position_fingerprint=fingerprint_metadata( + {"config": config, "effective_args": args} + ), + num_layers=args["num_hidden_layers"], + num_kv_heads=args.get("num_key_value_heads", args["num_attention_heads"]), + head_dim=args.get("head_dim") + or args["hidden_size"] // args["num_attention_heads"], + block_size=64, + logical_dtype="float16", + ) + return MLXXavierCache(contract, cache_config) + + +class MLXXavierCache: + def __init__(self, contract: KVCacheContract, config: dict): + self.contract, self.config = contract, dict(config) + self.role = config.get("role") + self._ref = None + self._configured = False + self._writes: set = set() + self.imported_tokens = 0 + + async def _call(self, method, *args): + async def invoke(): + if self._ref is None: + self._ref = await xo.actor_ref( + address=self.config["address"], uid=self.config["uid"] + ) + if not self._configured: + await self._ref.configure(self.contract.to_dict()) + self._configured = True + return await getattr(self._ref, method)(*args) + + return await asyncio.wait_for(invoke(), timeout=30) + + def keys(self, tokens): + keys = [key.digest for key in build_prefix_keys(self.contract, tokens)] + remainder = len(tokens) % self.contract.block_size + if remainder: + keys.append( + fingerprint_metadata( + { + "format": "xavier-partial-page-v1", + "contract": self.contract.fingerprint, + "previous": keys[-1] if keys else None, + "tokens": tokens[-remainder:], + "prefix_tokens": len(tokens), + } + ) + ) + return keys + + def encode(self, cache, tokens): + import mlx.core as mx + from mlx_lm.models.cache import KVCache + + c = self.contract + if len(cache) != c.num_layers: + raise ValueError("MLX cache layer count differs from its contract") + length = len(tokens) + if not length: + return [] + pages = (length + c.block_size - 1) // c.block_size + data = np.zeros( + (pages, c.num_layers, 2, c.block_size, c.num_kv_heads, c.head_dim), + dtype=" self.capacity_bytes: + raise ValueError("Xavier cache budget cannot hold one complete page") + self._contract = contract + self._page_bytes = page_bytes + self._capacity = self.capacity_bytes // page_bytes + return contract.fingerprint + + def _validate(self, namespace, keys): + if self._contract is None or namespace != self._contract.fingerprint: + raise ValueError("Unregistered or incompatible Xavier cache namespace") + if ( + not isinstance(keys, (list, tuple)) + or len(keys) > 4096 + or any( + not isinstance(key, str) + or len(key) != 64 + or any(c not in "0123456789abcdef" for c in key) + for key in keys + ) + or len(set(keys)) != len(keys) + ): + raise ValueError("Invalid Xavier page keys") + self._expire() + + def _expire(self): + for ticket, entry in list(self._handoffs.items()): + if entry["deadline"] <= time.monotonic(): + self.release_handoff(ticket) + + def _pinned(self): + return {key for entry in self._handoffs.values() for key in entry["keys"]} + + def put(self, namespace, keys, pages): + self._validate(namespace, keys) + if len(keys) != len(pages) or any( + type(page) is not bytes or len(page) != self._page_bytes for page in pages + ): + raise ValueError("Xavier page geometry differs from its contract") + pinned = self._pinned() + results = [] + for key, page in zip(keys, pages): + if key not in self._pages: + if len(self._pages) >= self._capacity: + victim = next((k for k in self._pages if k not in pinned), None) + if victim is None: + results.append(False) + continue + self._pages.pop(victim) + self._counts["evicted_pages"] += 1 + self._pages[key] = page + self._counts["stored_pages"] += 1 + # A published page is immutable, including while it is leased. + self._pages.move_to_end(key) + results.append(True) + return results + + def get(self, namespace, keys): + self._validate(namespace, keys) + pages = [] + for key in keys: + if key not in self._pages: + self._counts["missed_pages"] += 1 + break + pages.append(self._pages[key]) + self._pages.move_to_end(key) + self._counts["read_pages"] += len(pages) + return pages + + def prepare_handoff(self, namespace, keys): + self._validate(namespace, keys) + pinned = self._pinned().union(keys) + if len(pinned) > self._capacity: + raise RuntimeError("Xavier cache budget cannot hold the active PD prefixes") + # Reserve absent pages before P allocates or acknowledges a handoff. + while len(set(self._pages).union(pinned)) > self._capacity: + victim = next(k for k in self._pages if k not in pinned) + self._pages.pop(victim) + self._counts["evicted_pages"] += 1 + ticket = uuid.uuid4().hex + self._handoffs[ticket] = dict( + keys=list(keys), deadline=time.monotonic() + 300, claimed=False + ) + return ticket + + def get_handoff(self, namespace, keys, ticket): + self._validate(namespace, keys) + entry = self._handoffs.get(ticket) + if ( + entry is None + or entry["keys"] != list(keys) + or entry["claimed"] + or any(key not in self._pages for key in keys) + ): + raise ValueError( + "Missing, incomplete or already consumed Xavier PD handoff" + ) + entry["claimed"] = True + pages = [self._pages[key] for key in keys] + self._counts["read_pages"] += len(pages) + self._counts["handoff_reads"] += 1 + return pages + + def handoff_ready(self, namespace, keys, ticket): + self._validate(namespace, keys) + entry = self._handoffs.get(ticket) + if entry is None or entry["keys"] != list(keys) or entry["claimed"]: + raise ValueError("Missing or already consumed Xavier PD handoff") + return all(key in self._pages for key in keys) + + def release_handoff(self, ticket): + self._handoffs.pop(ticket, None) + + def get_stats(self): + self._expire() + return { + **self._counts, + "namespace": self._contract.fingerprint if self._contract else None, + "pages": len(self._pages), + "used_bytes": len(self._pages) * getattr(self, "_page_bytes", 0), + "capacity_bytes": self.capacity_bytes, + "active_handoffs": len(self._handoffs), + } diff --git a/xinference/model/llm/xavier/tests/test_bytes_storage.py b/xinference/model/llm/xavier/tests/test_bytes_storage.py new file mode 100644 index 0000000000..313bbdd709 --- /dev/null +++ b/xinference/model/llm/xavier/tests/test_bytes_storage.py @@ -0,0 +1,83 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +import pytest + +from ..backends.bytes.storage import XavierBytesCacheActor +from ..contract import KVCacheContract + + +def contract(): + return KVCacheContract( + *["0" * 64] * 4, + num_layers=2, + num_kv_heads=2, + head_dim=8, + block_size=64, + logical_dtype="float16", + ) + + +def store(capacity=2): + c = contract() + actor = XavierBytesCacheActor(capacity * c.layer_nbytes * c.num_layers) + namespace = actor.configure(c.to_dict()) + page = bytes(c.layer_nbytes * c.num_layers) + return actor, namespace, page + + +def test_capacity_geometry_and_contract(): + actor, namespace, page = store() + keys = [str(i) * 64 for i in range(1, 4)] + assert actor.put(namespace, keys, [page] * 3) == [True] * 3 + assert actor.get(namespace, keys) == [] + assert actor.get_stats()["pages"] == 2 + with pytest.raises(ValueError, match="geometry"): + actor.put(namespace, [keys[0]], [page[:-1]]) + with pytest.raises(ValueError, match="namespace"): + actor.get("wrong", keys) + metadata = contract().to_dict() + metadata["weights_fingerprint"] = "1" * 64 + with pytest.raises(ValueError, match="weights_fingerprint"): + actor.configure(metadata) + with pytest.raises(ValueError, match="budget"): + XavierBytesCacheActor(1).configure(contract().to_dict()) + + +def test_leases_pin_absent_pages_and_handoff_is_consumed_once(): + actor, namespace, page = store() + keys = ["1" * 64, "2" * 64] + ticket = actor.prepare_handoff(namespace, keys) + assert not actor.handoff_ready(namespace, keys, ticket) + with pytest.raises(ValueError, match="incomplete"): + actor.get_handoff(namespace, keys, ticket) + assert actor.put(namespace, keys, [page, page]) == [True, True] + assert actor.handoff_ready(namespace, keys, ticket) + assert actor.put(namespace, ["3" * 64], [page]) == [False] + with pytest.raises(RuntimeError, match="budget"): + actor.prepare_handoff(namespace, ["3" * 64]) + with pytest.raises(ValueError, match="incomplete"): + actor.get_handoff(namespace, [keys[0]], ticket) + assert actor.get_handoff(namespace, keys, ticket) == [page, page] + with pytest.raises(ValueError, match="consumed"): + actor.handoff_ready(namespace, keys, ticket) + with pytest.raises(ValueError, match="consumed"): + actor.get_handoff(namespace, keys, ticket) + actor.release_handoff(ticket) + actor.release_handoff(ticket) + assert actor.put(namespace, ["3" * 64], [page]) == [True] + assert actor.get_stats()["active_handoffs"] == 0 + + +def test_immutable_pages_expired_lease_and_empty_prefix(): + actor, namespace, page = store(1) + key = "1" * 64 + actor.put(namespace, [key], [page]) + actor.put(namespace, [key], [b"x" * len(page)]) + assert actor.get(namespace, [key]) == [page] + ticket = actor.prepare_handoff(namespace, [key]) + actor._handoffs[ticket]["deadline"] = 0 + assert actor.get_stats()["active_handoffs"] == 0 + ticket = actor.prepare_handoff(namespace, []) + assert actor.get_handoff(namespace, [], ticket) == [] + actor.release_handoff(ticket) + assert actor.get_stats()["handoff_reads"] == 1 From 8b6c2579eeaa16399ebebfb5622b99ef0a462ee0 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 00:02:45 +0800 Subject: [PATCH 2/4] fix: type internal MLX PD handoff metadata --- xinference/model/llm/mlx/core.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index e01f698583..f52f2cb285 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -1427,7 +1427,7 @@ async def async_generate( metadata = await xavier.prefill( self._model, tokens, generate_config.get("prompt_cache_prefix_len") ) - return Completion( + prefill_completion = Completion( id=str(uuid.uuid4()), object="text_completion", created=int(time.time()), @@ -1440,8 +1440,11 @@ async def async_generate( completion_tokens=0, total_tokens=len(tokens), ), - _pd_kv_transfer_params=metadata, ) + cast(Dict[str, Any], prefill_completion)[ + "_pd_kv_transfer_params" + ] = metadata + return prefill_completion if xavier.role == "decode" and ( not isinstance(transfer, dict) or not transfer.get("do_remote_prefill") ): @@ -1734,9 +1737,10 @@ async def _log_streaming_chunks(): c, "__aiter__" ), "async_generate should return Completion for non-streaming" c = cast(Completion, c) - if c.get("_pd_kv_transfer_params"): + transfer = cast(Dict[str, Any], c).get("_pd_kv_transfer_params") + if transfer: result = self._to_chat_completion(c, self.reasoning_parser) - result["_pd_kv_transfer_params"] = c["_pd_kv_transfer_params"] + cast(Dict[str, Any], result)["_pd_kv_transfer_params"] = transfer return result if tools: return self._post_process_completion( From 8d88986eb3b83602b636d857386f327d8034861b Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 23:19:57 +0800 Subject: [PATCH 3/4] fix: repair MLX Xavier recovery and prefix publication --- xinference/core/supervisor.py | 13 +- xinference/core/tests/test_pd_launch.py | 63 +++- xinference/core/worker.py | 5 +- xinference/model/llm/mlx/core.py | 36 ++- xinference/model/llm/mlx/tests/test_xavier.py | 282 +++++++++++++++++- xinference/model/llm/mlx/xavier.py | 109 ++++--- .../llm/xavier/backends/bytes/storage.py | 32 +- .../llm/xavier/tests/test_bytes_storage.py | 38 ++- 8 files changed, 504 insertions(+), 74 deletions(-) diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index 50721d06bd..703b7cf44e 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3112,6 +3112,14 @@ async def launch_builtin_model( raise ValueError("NIXL requires explicit prefill and decode replica roles") # Xavier-related requested_xavier = bool(kwargs.pop("enable_xavier", False)) + if ( + requested_xavier + and not pd_enabled + and replica <= 1 + and (model_engine or "").lower() in ("mlx", "sglang") + ): + logger.warning("Enabling xavier when replica<=1 is meaningless.") + requested_xavier = False mlx_xavier = ( (requested_xavier or pd_enabled) and transport_backend == "xavier" @@ -3125,7 +3133,7 @@ async def launch_builtin_model( if ( model_type not in (None, "LLM") or model_format != "mlx" - or quantization not in (None, "none") + or quantization not in (None, "none", "fp16", "bf16") ): raise ValueError("MLX Xavier requires unquantized MLX text weights") kwargs["_xavier_cache_config"] = { @@ -3138,9 +3146,6 @@ async def launch_builtin_model( and model_engine is not None and model_engine.lower() == "sglang" ) - if sglang_xavier and not pd_enabled and replica <= 1: - logger.warning("Enabling xavier when replica<=1 is meaningless.") - sglang_xavier = False sglang_nixl = ( pd_enabled and transport_backend == "nixl" diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index 32aa00f216..18d51d22a4 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -69,10 +69,16 @@ def launch_kwargs(): @pytest.mark.asyncio @pytest.mark.parametrize("pd", [False, True]) -async def test_mlx_xavier_launch_and_cleanup(launch_runtime, pd): +@pytest.mark.parametrize("quantization", [None, "none", "fp16", "bf16"]) +async def test_mlx_xavier_launch_and_cleanup(launch_runtime, pd, quantization): supervisor, workers, actors, destroy = launch_runtime kwargs = launch_kwargs() - kwargs.update(model_engine="MLX", model_format="mlx", xavier_cache_bytes=123456) + kwargs.update( + model_engine="MLX", + model_format="mlx", + quantization=quantization, + xavier_cache_bytes=123456, + ) if not pd: kwargs["enable_xavier"] = True for replica in kwargs["replica_config"]: @@ -99,6 +105,59 @@ async def test_mlx_xavier_launch_and_cleanup(launch_runtime, pd): assert destroy.await_count == (2 if pd else 1) +@pytest.mark.asyncio +async def test_single_mlx_replica_disables_shared_xavier(launch_runtime, caplog): + supervisor, workers, actors, _ = launch_runtime + supervisor._resolve_replica_config.return_value = ([(workers[0], [0], 1)], {0: "p"}) + kwargs = launch_kwargs() + kwargs.update(model_engine="MLX", model_format="mlx", enable_xavier=True, replica=1) + kwargs["replica_config"] = kwargs["replica_config"][:1] + kwargs["replica_config"][0].role = None + await supervisor.launch_builtin_model(**kwargs) + assert "replica<=1" in caplog.text + assert not actors and not supervisor._xavier_cache_mapping + assert ( + "_xavier_cache_config" not in workers[0].launch_builtin_model.call_args.kwargs + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("role", ["prefill", "decode"]) +async def test_mlx_pd_recovery_keeps_bytes_cache_and_registers_replacement(role): + import xoscar as xo + + from ...model.llm.xavier.backends.bytes.storage import XavierBytesCacheActor + from ...model.llm.xavier.tests.test_bytes_storage import contract + from ..worker import WorkerActor + + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + cache = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="bytes-cache" + ) + namespace = await cache.configure(contract().to_dict()) + worker = MagicMock() + supervisor = AsyncMock() + worker.get_supervisor_ref = AsyncMock(return_value=supervisor) + worker.launch_builtin_model = AsyncMock(return_value="replacement:1234") + worker.wait_for_load = AsyncMock() + replacement = MagicMock() + worker._model_uid_to_model = {"pd-rep0": replacement} + config = dict(role=role, address=cache.address, uid=cache.uid) + await WorkerActor.recover_model( + worker, dict(model_uid="pd-rep0", _xavier_cache_config=config) + ) + worker.launch_builtin_model.assert_awaited_once_with( + model_uid="pd-rep0", _xavier_cache_config=config + ) + worker.wait_for_load.assert_awaited_once_with("pd-rep0") + supervisor.unregister_pd_replica.assert_awaited_once_with("pd", "pd-rep0") + supervisor.register_pd_replica.assert_awaited_once_with( + "pd", "pd-rep0", replacement + ) + assert (await cache.get_stats())["namespace"] == namespace + + @pytest.mark.asyncio async def test_mlx_failed_launch_cleans_bytes_actor(launch_runtime): supervisor, workers, actors, destroy = launch_runtime diff --git a/xinference/core/worker.py b/xinference/core/worker.py index 37315e8fc8..1b75363794 100644 --- a/xinference/core/worker.py +++ b/xinference/core/worker.py @@ -6903,7 +6903,10 @@ async def recover_model(self, launch_args: Dict[str, Any]): ).get("role") in ("prefill", "decode"): await supervisor_ref.unregister_pd_replica(origin_uid, rep_model_uid) cache_config = launch_args.get("_xavier_cache_config", {}) - if cache_config.get("role") in ("prefill", "decode"): + if ( + cache_config.get("role") in ("prefill", "decode") + and "rank" in cache_config + ): directory = await xo.actor_ref( address=cache_config["address"], uid=cache_config["uid"] ) diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index f52f2cb285..2afe36b676 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -226,6 +226,7 @@ def _get_or_create_generator( "active": set(), # active uids "cache_boundaries": {}, # uid -> stable prefix token count "xavier_prompts": {}, + "xavier_writes": {}, "task": None, } @@ -345,11 +346,16 @@ def _store_xavier_prompt_caches(self, batch_generator, prompt_results, gen_dict) for result in prompt_results: if not result.end_of_prompt or result.uid not in prompts: continue - extracted = batch_generator.extract_cache([result.uid]).get(result.uid) - if extracted is not None: - cache, _ = extracted - tokens = prompts.pop(result.uid) - self._xavier.publish(cache, tokens[:-1]) + tokens, cached_tokens = prompts.pop(result.uid) + try: + extracted = batch_generator.extract_cache([result.uid]).get(result.uid) + if extracted is not None: + cache, _ = extracted + gen_dict["xavier_writes"][result.uid] = self._xavier.publish( + cache, tokens[:-1], cached_tokens + ) + except Exception: + logger.warning("MLX Xavier prefix publication failed", exc_info=True) async def _background_worker(self, gen_dict): """Background worker that continuously calls next() and distributes results.""" @@ -425,7 +431,6 @@ async def generate_stream( request_id: Optional[str] = None, skip_special_tokens: bool = True, prompt_cache_prefix_len: Optional[int] = None, - kv_transfer_params: Optional[dict] = None, prepared_cache=None, prompt_token_ids: Optional[List[int]] = None, ) -> AsyncGenerator[CompletionChunk, None]: @@ -450,8 +455,6 @@ async def generate_stream( queue: asyncio.Queue = asyncio.Queue() external_cache = prepared_cache - if external_cache is None and getattr(self, "_xavier", None) is not None: - external_cache = await self._xavier.fetch(prompt_tokens, kv_transfer_params) insert_args = ( batch_generator, prompt_tokens, @@ -463,11 +466,15 @@ async def generate_stream( if external_cache is not None else self._insert_request(*insert_args) ) + remote_cached_tokens = external_cache[1] if external_cache is not None else 0 if ( getattr(self, "_xavier", None) is not None - and cached_prompt_tokens < input_echo_len - 1 + and remote_cached_tokens < input_echo_len - 1 ): - gen_dict["xavier_prompts"][inserted_uid] = prompt_tokens + gen_dict["xavier_prompts"][inserted_uid] = ( + prompt_tokens, + remote_cached_tokens, + ) if cache_boundary is not None: gen_dict["cache_boundaries"][inserted_uid] = cache_boundary @@ -606,7 +613,8 @@ async def generate_stream( gen_dict["xavier_prompts"].pop(inserted_uid, None) batch_generator.remove([inserted_uid]) if getattr(self, "_xavier", None) is not None: - await self._xavier.flush() + write = gen_dict["xavier_writes"].pop(inserted_uid, None) + await self._xavier.flush([write] if write is not None else []) async def generate( self, @@ -618,7 +626,6 @@ async def generate( stream: bool = False, skip_special_tokens: bool = True, prompt_cache_prefix_len: Optional[int] = None, - kv_transfer_params: Optional[dict] = None, prepared_cache=None, prompt_token_ids: Optional[List[int]] = None, ) -> Tuple[str, CompletionUsage]: @@ -640,7 +647,6 @@ async def generate( request_id=None, skip_special_tokens=skip_special_tokens, prompt_cache_prefix_len=prompt_cache_prefix_len, - kv_transfer_params=kv_transfer_params, prepared_cache=prepared_cache, prompt_token_ids=prompt_token_ids, ): @@ -1119,6 +1125,10 @@ def wait_for_load(self): raise ValueError( "MLX Xavier requires the continuous batching text engine" ) + assert self._loop is not None, "Service not started correctly" + asyncio.run_coroutine_threadsafe( + self._xavier.initialize(), self._loop + ).result() # Update allow_batch based on distributed inference # Only enable continuous batching for non-distributed inference (single worker) diff --git a/xinference/model/llm/mlx/tests/test_xavier.py b/xinference/model/llm/mlx/tests/test_xavier.py index d3e824c784..1c228ef8d0 100644 --- a/xinference/model/llm/mlx/tests/test_xavier.py +++ b/xinference/model/llm/mlx/tests/test_xavier.py @@ -94,6 +94,266 @@ async def test_decode_claims_before_returning_lazy_stream(): await stream.aclose() +@pytest.mark.asyncio +async def test_decode_passes_prepared_cache_and_tokens_to_batch_stream(): + from ..core import MLXModel + + wrapper = object.__new__(MLXModel) + wrapper._model = object() + wrapper._tokenizer = SimpleNamespace(encode=lambda prompt: [1, 2, 3]) + prepared = ([object()], 2) + wrapper._xavier = SimpleNamespace( + role="decode", fetch=AsyncMock(return_value=prepared) + ) + wrapper._model_generation_config = {} + wrapper.model_uid = "mlx" + + async def chunks(**kwargs): + yield {"choices": [{"text": "token"}]} + + wrapper._batch_model = SimpleNamespace( + generate_stream=MagicMock(side_effect=chunks) + ) + transfer = {"do_remote_prefill": True} + stream = await wrapper.async_generate( + "prompt", {"stream": True, "max_tokens": 2, "_pd_kv_transfer_params": transfer} + ) + assert [c["model"] async for c in stream] == ["mlx"] + kwargs = wrapper._batch_model.generate_stream.call_args.kwargs + assert kwargs["prepared_cache"] is prepared and kwargs["prompt_token_ids"] == [ + 1, + 2, + 3, + ] + wrapper._xavier.fetch.assert_awaited_once_with([1, 2, 3], transfer) + + +@pytest.mark.asyncio +async def test_chat_prefill_preserves_handoff_metadata(): + from ..core import MLXChatModel + + wrapper = object.__new__(MLXChatModel) + wrapper._model = object() + wrapper._tokenizer = SimpleNamespace(encode=lambda prompt: [1, 2, 3]) + wrapper._model_generation_config = {} + wrapper.model_uid = "mlx" + wrapper.model_family = SimpleNamespace( + model_family="qwen2", + model_name="qwen2", + chat_template="template", + stop=None, + stop_token_ids=None, + ) + wrapper.reasoning_parser = None + wrapper.get_full_context = MagicMock(return_value="chat prompt") + wrapper._get_reusable_prompt_prefix_len = MagicMock(return_value=2) + transfer = {"do_remote_prefill": True, "mlx_xavier": {"ticket": "ticket"}} + wrapper._xavier = SimpleNamespace( + role="prefill", prefill=AsyncMock(return_value=transfer) + ) + result = await wrapper.async_chat( + [{"role": "user", "content": "hello"}], + {"_pd_kv_transfer_params": {"do_remote_decode": True}}, + ) + assert ( + result["object"] == "chat.completion" + and result["_pd_kv_transfer_params"] is transfer + ) + wrapper._xavier.prefill.assert_awaited_once_with(wrapper._model, [1, 2, 3], 2) + + +@pytest.mark.asyncio +async def test_publication_failure_does_not_drop_batch_results(monkeypatch, caplog): + from ..core import MLXBatchModel + + monkeypatch.setattr(MLXBatchModel, "_lock", None) + monkeypatch.setattr(MLXBatchModel, "_is_new_mlx_lm", lambda: True) + wrapper = object.__new__(MLXBatchModel) + wrapper._prompt_cache = None + wrapper._xavier = SimpleNamespace( + publish=MagicMock(side_effect=[ValueError("bad cache"), None]) + ) + prompts = [SimpleNamespace(uid=i, end_of_prompt=True) for i in (1, 2)] + results = [SimpleNamespace(uid=i, finish_reason="stop", token=7) for i in (1, 2)] + generator = SimpleNamespace( + sampler="test", + next=MagicMock(side_effect=[(prompts, results)]), + extract_cache=lambda uids: {uids[0]: ([], [])}, + ) + queues = {i: asyncio.Queue() for i in (1, 2)} + state = dict( + generator=generator, + queues=queues, + pending={}, + active={1, 2}, + cache_boundaries={}, + xavier_prompts={i: ([1, 2], 0) for i in (1, 2)}, + xavier_writes={}, + ) + task = asyncio.create_task(wrapper._background_worker(state)) + try: + delivered = await asyncio.wait_for( + asyncio.gather(*(q.get() for q in queues.values())), 1 + ) + assert delivered == results and not state["active"] + assert wrapper._xavier.publish.call_count == 2 + assert "prefix publication failed" in caplog.text + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_cancelled_flush_keeps_other_requests_writes_independent(): + client = MLXXavierCache(contract(), {}) + client.encode = MagicMock(return_value=[b"page"]) + gates = {client.keys([i])[0]: asyncio.Event() for i in (1, 2)} + + async def put(method, namespace, keys, pages, start): + await gates[keys[0]].wait() + + client._call = put + first, second = (client.publish([], [i]) for i in (1, 2)) + flush = asyncio.create_task(client.flush([first])) + await asyncio.sleep(0) + flush.cancel() + with pytest.raises(asyncio.CancelledError): + await flush + assert not first.cancelled() and not second.cancelled() + gates[client.keys([2])[0]].set() + await asyncio.wait_for(client.flush([second]), 1) + assert not first.done() + gates[client.keys([1])[0]].set() + await asyncio.wait_for(client.flush([first]), 1) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("decode_error", [False, True]) +async def test_failed_release_keeps_decode_result_or_original_error( + decode_error, caplog +): + cfg = dict(address="address", uid="cache", role="decode") + client = MLXXavierCache(contract(), cfg) + tokens = [1, 2, 3] + transfer = { + "mlx_xavier": dict( + engine="mlx", + address="address", + uid="cache", + namespace=client.contract.fingerprint, + keys=client.keys(tokens[:-1]), + tokens=2, + ticket="ticket", + ) + } + client._call = AsyncMock(side_effect=[[b"page"], TimeoutError("release failed")]) + client.decode = MagicMock( + side_effect=ValueError("decode failed") if decode_error else None, + return_value="cache", + ) + if decode_error: + with pytest.raises(ValueError, match="decode failed"): + await client.fetch(tokens, transfer) + else: + assert await client.fetch(tokens, transfer) == ("cache", 2) + assert "handoff release failed" in caplog.text + + +@metal +@pytest.mark.asyncio +async def test_load_registers_contract_and_rejects_incompatible_replica(tmp_path): + from ..core import MLXModel + + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="mlx-cache" + ) + for i in (0, 1): + path = tmp_path / str(i) + path.mkdir() + (path / "config.json").write_text( + json.dumps({"model_type": "qwen2", "max_position_embeddings": 2048}) + ) + (path / "tokenizer.json").write_text("{}") + (path / "model.safetensors").write_bytes(bytes([i])) + wrapper = object.__new__(MLXModel) + wrapper._model = model() + wrapper._loading_thread = None + wrapper._model_config = {} + wrapper._xavier_config = dict( + address=ref.address, uid=ref.uid, role="prefill" if i == 0 else "decode" + ) + wrapper._n_worker = 1 + wrapper._batch_model = object() + wrapper.allow_batch = True + wrapper._loop = asyncio.get_running_loop() + wrapper.model_path = str(path) + wrapper._update_model_generation_config = MagicMock() + if i == 0: + await asyncio.to_thread(wrapper.wait_for_load) + assert (await ref.get_stats())[ + "namespace" + ] == wrapper._xavier.contract.fingerprint + else: + with pytest.raises(ValueError, match="weights_fingerprint"): + await asyncio.to_thread(wrapper.wait_for_load) + + +@metal +def test_weight_fingerprint_cache_avoids_rehash_and_invalidates_changes( + tmp_path, monkeypatch +): + from pathlib import Path + + import xinference.model.llm.mlx.xavier as module + + monkeypatch.setattr(module, "XINFERENCE_CACHE_DIR", str(tmp_path / "cache")) + (tmp_path / "config.json").write_text(json.dumps({"model_type": "qwen2"})) + (tmp_path / "tokenizer.json").write_text("{}") + weight = tmp_path / "model.safetensors" + weight.write_bytes(b"first") + reads = [] + saved = Path.open + + def tracked(path, mode="r", *args, **kwargs): + if path == weight and mode == "rb": + reads.append(path) + return saved(path, mode, *args, **kwargs) + + monkeypatch.setattr(Path, "open", tracked) + first = configure_xavier(model(), tmp_path, {}, {}, 1).contract + second = configure_xavier(model(), tmp_path, {}, {}, 1).contract + assert first.weights_fingerprint == second.weights_fingerprint and len(reads) == 1 + weight.write_bytes(b"changed") + third = configure_xavier(model(), tmp_path, {}, {}, 1).contract + assert third.weights_fingerprint != first.weights_fingerprint and len(reads) == 2 + + +@metal +@pytest.mark.asyncio +@pytest.mark.parametrize("cached", [0, 64, 65, 128]) +async def test_publish_encodes_only_pages_beyond_remote_prefix(cached): + import mlx.core as mx + from mlx_lm.models.cache import make_prompt_cache + + m = model() + tokens = [i % 128 for i in range(130)] + cache = make_prompt_cache(m) + mx.eval(m(mx.array(tokens)[None], cache=cache)) + client = MLXXavierCache(contract(), {}) + complete = client.encode(cache, tokens) + client._call = AsyncMock() + await client.publish(cache, tokens, cached_tokens=cached) + client._call.assert_awaited_once_with( + "put", + client.contract.fingerprint, + client.keys(tokens), + complete[cached // 64 :], + cached // 64, + ) + + @metal @pytest.mark.parametrize( "config", @@ -232,7 +492,8 @@ async def test_real_pd_import_skips_prefill_and_matches_logits(): @metal @pytest.mark.asyncio -async def test_pd_failed_prefill_releases_reserved_pages(): +@pytest.mark.parametrize("release_fails", [False, True]) +async def test_pd_failed_prefill_releases_reserved_pages(release_fails): pool = await xo.create_actor_pool("127.0.0.1", n_process=0) async with pool: ref = await xo.create_actor( @@ -241,6 +502,15 @@ async def test_pd_failed_prefill_releases_reserved_pages(): p = MLXXavierCache( contract(), dict(address=ref.address, uid="mlx-cache", role="prefill") ) + if release_fails: + original_call = p._call + + async def call(method, *args): + if method == "release_handoff": + raise TimeoutError("cleanup failed") + return await original_call(method, *args) + + p._call = call m = model() def fail(*args, **kwargs): @@ -252,7 +522,7 @@ def fail(*args, **kwargs): await p.prefill(m, [1, 2, 3]) finally: m.__class__.__call__ = saved - assert (await ref.get_stats())["active_handoffs"] == 0 + assert (await ref.get_stats())["active_handoffs"] == int(release_fails) @metal @@ -287,8 +557,14 @@ async def generate(role=None, transfer=None): model(), Tokenizer(), prompt_cache_size=0, xavier=client ) try: + tokens = Tokenizer().encode("a" * 70) + prepared = await client.fetch(tokens, transfer) return await wrapper.generate( - "a" * 70, 8, temperature=0, kv_transfer_params=transfer + "a" * 70, + 8, + temperature=0, + prepared_cache=prepared, + prompt_token_ids=tokens, ) finally: tasks = [ diff --git a/xinference/model/llm/mlx/xavier.py b/xinference/model/llm/mlx/xavier.py index b3c556beff..4acfe328b8 100644 --- a/xinference/model/llm/mlx/xavier.py +++ b/xinference/model/llm/mlx/xavier.py @@ -12,6 +12,7 @@ import numpy as np import xoscar as xo +from ....constants import XINFERENCE_CACHE_DIR from ..xavier.contract import ( KVCacheContract, build_prefix_keys, @@ -20,6 +21,17 @@ ) logger = logging.getLogger(__name__) +_TOKENIZER_FILES = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "tokenizer.model", + "spiece.model", +) +_TOKENIZER_ASSETS = ("tokenizer.json", "tokenizer.model", "spiece.model", "vocab.json") def configure_xavier(model, model_path, model_config, cache_config, n_worker): @@ -58,25 +70,9 @@ def configure_xavier(model, model_path, model_config, cache_config, n_worker): if not all(type(c) is KVCache for c in make_prompt_cache(model)): raise ValueError("MLX Xavier requires ordinary full-attention KVCache layers") tokenizer_files = { - p.name: p - for p in path.iterdir() - if p.name - in ( - "tokenizer.json", - "tokenizer_config.json", - "special_tokens_map.json", - "added_tokens.json", - "vocab.json", - "merges.txt", - "tokenizer.model", - "spiece.model", - ) - and p.is_file() + p.name: p for p in path.iterdir() if p.name in _TOKENIZER_FILES and p.is_file() } - if not tokenizer_files or not any( - n in tokenizer_files - for n in ("tokenizer.json", "tokenizer.model", "spiece.model", "vocab.json") - ): + if not any(n in tokenizer_files for n in _TOKENIZER_ASSETS): raise ValueError("MLX Xavier requires local tokenizer assets") tokenizer_config = ( json.loads(tokenizer_files["tokenizer_config.json"].read_text()) @@ -91,7 +87,9 @@ def configure_xavier(model, model_path, model_config, cache_config, n_worker): model.set_dtype(mx.float16) args = asdict(model.args) contract = KVCacheContract( - weights_fingerprint=fingerprint_files(weights), + weights_fingerprint=fingerprint_files( + weights, cache_dir=Path(XINFERENCE_CACHE_DIR) / "xavier-fingerprints" + ), tokenizer_fingerprint=fingerprint_files(tokenizer_files), attention_fingerprint=fingerprint_metadata( {"config": config, "effective_args": args, "dtype": "float16"} @@ -118,15 +116,21 @@ def __init__(self, contract: KVCacheContract, config: dict): self._writes: set = set() self.imported_tokens = 0 + async def _ensure_configured(self): + if self._ref is None: + self._ref = await xo.actor_ref( + address=self.config["address"], uid=self.config["uid"] + ) + if not self._configured: + await self._ref.configure(self.contract.to_dict()) + self._configured = True + + async def initialize(self): + await asyncio.wait_for(self._ensure_configured(), timeout=30) + async def _call(self, method, *args): async def invoke(): - if self._ref is None: - self._ref = await xo.actor_ref( - address=self.config["address"], uid=self.config["uid"] - ) - if not self._configured: - await self._ref.configure(self.contract.to_dict()) - self._configured = True + await self._ensure_configured() return await getattr(self._ref, method)(*args) return await asyncio.wait_for(invoke(), timeout=30) @@ -148,7 +152,7 @@ def keys(self, tokens): ) return keys - def encode(self, cache, tokens): + def encode(self, cache, tokens, start=0): import mlx.core as mx from mlx_lm.models.cache import KVCache @@ -156,9 +160,11 @@ def encode(self, cache, tokens): if len(cache) != c.num_layers: raise ValueError("MLX cache layer count differs from its contract") length = len(tokens) - if not length: + if start % c.block_size or not 0 <= start <= length: + raise ValueError("MLX page publication must start at a block boundary") + if start == length: return [] - pages = (length + c.block_size - 1) // c.block_size + pages = (length - start + c.block_size - 1) // c.block_size data = np.zeros( (pages, c.num_layers, 2, c.block_size, c.num_kv_heads, c.head_dim), dtype=" self._capacity: + return [False] * len(keys) pinned = self._pinned() - results = [] - for key, page in zip(keys, pages): + for key, page in reversed(list(zip(keys[start:], pages))): if key not in self._pages: if len(self._pages) >= self._capacity: victim = next((k for k in self._pages if k not in pinned), None) if victim is None: - results.append(False) continue self._pages.pop(victim) self._counts["evicted_pages"] += 1 self._pages[key] = page self._counts["stored_pages"] += 1 - # A published page is immutable, including while it is leased. - self._pages.move_to_end(key) - results.append(True) - return results + # Touch the whole chain, including its already cached prefix, so suffix + # publication also keeps heads newer than tails. Pages remain immutable. + for key in reversed(keys): + if key in self._pages: + self._pages.move_to_end(key) + return [key in self._pages for key in keys] def get(self, namespace, keys): self._validate(namespace, keys) @@ -97,6 +105,8 @@ def get(self, namespace, keys): self._counts["missed_pages"] += 1 break pages.append(self._pages[key]) + # Keep chain heads newer than tails so eviction preserves usable prefixes. + for key in reversed(keys[: len(pages)]): self._pages.move_to_end(key) self._counts["read_pages"] += len(pages) return pages @@ -131,6 +141,8 @@ def get_handoff(self, namespace, keys, ticket): ) entry["claimed"] = True pages = [self._pages[key] for key in keys] + for key in reversed(keys): + self._pages.move_to_end(key) self._counts["read_pages"] += len(pages) self._counts["handoff_reads"] += 1 return pages diff --git a/xinference/model/llm/xavier/tests/test_bytes_storage.py b/xinference/model/llm/xavier/tests/test_bytes_storage.py index 313bbdd709..6a23baa9f0 100644 --- a/xinference/model/llm/xavier/tests/test_bytes_storage.py +++ b/xinference/model/llm/xavier/tests/test_bytes_storage.py @@ -28,8 +28,9 @@ def store(capacity=2): def test_capacity_geometry_and_contract(): actor, namespace, page = store() keys = [str(i) * 64 for i in range(1, 4)] - assert actor.put(namespace, keys, [page] * 3) == [True] * 3 + assert actor.put(namespace, keys, [page] * 3) == [False] * 3 assert actor.get(namespace, keys) == [] + assert actor.put(namespace, keys[:2], [page] * 2) == [True] * 2 assert actor.get_stats()["pages"] == 2 with pytest.raises(ValueError, match="geometry"): actor.put(namespace, [keys[0]], [page[:-1]]) @@ -43,6 +44,41 @@ def test_capacity_geometry_and_contract(): XavierBytesCacheActor(1).configure(contract().to_dict()) +@pytest.mark.parametrize("touch", [False, True]) +def test_eviction_preserves_chain_head_and_oversized_put_preserves_cache(touch): + actor, namespace, page = store(3) + keys = [str(i) * 64 for i in range(1, 5)] + actor.put(namespace, keys[:3], [page] * 3) + if touch: + assert actor.get(namespace, keys[:3]) == [page] * 3 + actor.put(namespace, keys[3:], [page]) + assert actor.get(namespace, keys[:3]) == [page] * 2 + assert actor.get(namespace, keys[3:]) == [page] + assert actor.put(namespace, keys, [page] * 4) == [False] * 4 + assert actor.get(namespace, keys[:3]) == [page] * 2 + + +def test_put_reports_pages_that_survive_pinned_capacity_pressure(): + actor, namespace, page = store(3) + keys = [str(i) * 64 for i in range(1, 5)] + ticket = actor.prepare_handoff(namespace, keys[3:]) + actor.put(namespace, keys[3:], [page]) + assert actor.put(namespace, keys[:3], [page] * 3) == [True, True, False] + assert actor.get(namespace, keys[:3]) == [page] * 2 + assert actor.get_handoff(namespace, keys[3:], ticket) == [page] + + +def test_suffix_publication_keeps_existing_chain_heads_newer_than_tails(): + actor, namespace, page = store(4) + keys = [str(i) * 64 for i in range(1, 6)] + actor.put(namespace, keys[:2], [page] * 2) + assert actor.put(namespace, keys[:4], [page] * 2, start=2) == [True] * 4 + actor.put(namespace, keys[4:], [page]) + assert actor.get(namespace, keys[:4]) == [page] * 3 + with pytest.raises(ValueError, match="geometry"): + actor.put(namespace, keys[:4], [page], start=2) + + def test_leases_pin_absent_pages_and_handoff_is_consumed_once(): actor, namespace, page = store() keys = ["1" * 64, "2" * 64] From 7a329378b83e985832865b60eb9d4e0e9f1119ad Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 00:25:27 +0800 Subject: [PATCH 4/4] fix: honor MLX Xavier capacity and refresh aged prefixes --- .../LC_MESSAGES/user_guide/pd_separation.po | 45 +++++++--- .../LC_MESSAGES/user_guide/pd_separation.po | 39 +++++++-- .../LC_MESSAGES/user_guide/pd_separation.po | 41 ++++++++-- .../LC_MESSAGES/user_guide/pd_separation.po | 45 +++++++--- .../LC_MESSAGES/user_guide/pd_separation.po | 35 ++++++-- .../LC_MESSAGES/user_guide/pd_separation.po | 37 +++++++-- .../LC_MESSAGES/user_guide/pd_separation.po | 42 +++++++--- .../LC_MESSAGES/user_guide/pd_separation.po | 34 ++++++-- .../LC_MESSAGES/user_guide/pd_separation.po | 34 ++++++-- doc/source/user_guide/pd_separation.rst | 6 +- xinference/core/supervisor.py | 6 +- xinference/core/tests/test_pd_launch.py | 3 +- xinference/model/llm/mlx/core.py | 2 + xinference/model/llm/mlx/tests/test_xavier.py | 82 +++++++++++++++++++ xinference/model/llm/mlx/xavier.py | 25 ++++-- .../llm/xavier/backends/bytes/storage.py | 11 ++- .../llm/xavier/tests/test_bytes_storage.py | 29 ++++++- 17 files changed, 415 insertions(+), 101 deletions(-) diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po index 6153d5ac3c..145403836a 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po @@ -59,16 +59,16 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "Dieselbe ``replica_config`` wird von ``POST /v1/models``, dem asynchronen " "Python-Client und der CLI-JSON-Option ``--replica_config`` akzeptiert. " -"Wählen Sie in der Web UI vLLM oder SGLang, aktivieren Sie die Platzierung " -"pro Replikat und wählen Sie Prefill oder Decode für jedes Replikat. Der " -"Community-Startpfad verwendet standardmäßig Xavier, sobald P/D-Rollen " -"angegeben werden." +"Wählen Sie in der Web UI vLLM, SGLang oder MLX, aktivieren Sie die " +"Platzierung pro Replikat und wählen Sie Prefill oder Decode für jedes " +"Replikat. Der Community-Startpfad verwendet standardmäßig Xavier, sobald " +"P/D-Rollen angegeben werden." #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -483,8 +483,9 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier speichert unveränderliche CPU-Seiten in einem gemeinsamen Actor " "auf dem Supervisor und überträgt sie per Actor-RPC. ``xavier_cache_bytes`` " @@ -494,8 +495,9 @@ msgstr "" " berechnet das letzte Token und wählt das erste Antwort-Token. Streaming- " "und Nicht-Streaming-Anfragen werden unterstützt. Aktive Übergaben " "reservieren Cache-Kapazität; unzureichende Kapazität, inkompatible Metadaten" -" oder Übertragungsfehler lösen einen Fehler aus. Nicht abgeholte Übergaben " -"verfallen nach fünf Minuten." +" oder Übertragungsfehler lösen einen Fehler aus. Die fünfminütige " +"Übergabefrist beginnt, sobald Prefill Kapazität reserviert, vor der " +"Berechnung des Präfixes, und umfasst Prefill und Übertragung." #: user_guide/pd_separation.rst msgid "" @@ -511,3 +513,24 @@ msgstr "" "Replikate auf einem Mac teilen dessen Metal-GPU; messen Sie Ihren Workload, " "bevor Sie einen Durchsatzvorteil erwarten. Engineübergreifendes P/D " "einschließlich NVIDIA-zu-Mac-Übergaben wird noch nicht unterstützt." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"Jede FP16-Seite mit 64 Tokens benötigt ``2 * 64 * num_layers * num_kv_heads " +"* head_dim * 2`` Bytes (K und V mit zwei Bytes pro Wert). Beispielsweise " +"benötigen 36 Schichten, 8 KV-Köpfe und eine Kopfdimension von 128 pro Seite " +"9 MiB; 512 MiB reichen für 56 Seiten oder 3584 Präfix-Tokens. Runden Sie " +"jedes Präfix auf eine ganze Seite auf und dimensionieren Sie " +"``xavier_cache_bytes`` für die unterschiedlichen Seiten, die alle " +"gleichzeitigen P/D-Anfragen reservieren. Gewöhnliche Anfragen mit " +"gemeinsamem Cache überspringen die Veröffentlichung, wenn ihr Präfix die " +"Cache-Kapazität oder die Protokollgrenze von 4096 Seiten überschreitet." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po index 5b1fd16b86..3068bce31c 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po @@ -59,13 +59,13 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "El mismo ``replica_config`` se acepta en ``POST /v1/models``, el cliente " "Python asíncrono y la opción JSON ``--replica_config`` de la CLI. En la Web " -"UI, seleccione vLLM o SGLang, active la colocación por réplica y elija " +"UI, seleccione vLLM, SGLang o MLX, active la colocación por réplica y elija " "Prefill o Decode para cada réplica. El inicio de la edición comunitaria usa " "Xavier por defecto cuando se indican roles P/D." @@ -478,8 +478,9 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier guarda páginas CPU inmutables en un actor compartido del " "supervisor y las transfiere por RPC de actores. ``xavier_cache_bytes`` " @@ -489,8 +490,9 @@ msgstr "" "prefijo, calcula ese último token y muestrea el primer token de respuesta. " "Se admiten solicitudes con y sin streaming. Las transferencias activas " "reservan capacidad de caché; capacidad insuficiente, metadatos incompatibles" -" o fallos de transferencia generan un error. Las transferencias no recogidas" -" caducan a los cinco minutos." +" o fallos de transferencia generan un error. El plazo de cinco minutos " +"comienza cuando Prefill reserva capacidad, antes de calcular el prefijo, e " +"incluye el tiempo de prefill y transferencia." #: user_guide/pd_separation.rst msgid "" @@ -506,3 +508,24 @@ msgstr "" " un Mac comparten su GPU Metal; mida su carga antes de esperar mejoras de " "rendimiento. El P/D entre motores, incluida la transferencia de NVIDIA a " "Mac, aún no es compatible." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"Cada página FP16 de 64 tokens utiliza ``2 * 64 * num_layers * num_kv_heads *" +" head_dim * 2`` bytes (K y V, con dos bytes por valor). Por ejemplo, 36 " +"capas, 8 cabezas KV y una dimensión de cabeza de 128 requieren 9 MiB por " +"página; 512 MiB almacenan 56 páginas, o 3584 tokens de prefijo. Redondee " +"cada prefijo a una página completa y dimensione ``xavier_cache_bytes`` para " +"las páginas distintas reservadas por todas las solicitudes P/D simultáneas. " +"Las solicitudes ordinarias de caché compartida omiten la publicación cuando " +"su prefijo supera la capacidad de caché o el límite de protocolo de 4096 " +"páginas." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po index 65fb7b975c..a26fa13e26 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po @@ -59,14 +59,14 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "Le même ``replica_config`` est accepté par ``POST /v1/models``, le client " "Python asynchrone et l’option JSON ``--replica_config`` de la CLI. Dans la " -"Web UI, sélectionnez vLLM ou SGLang, activez le placement par réplique et " -"choisissez Prefill ou Decode pour chaque réplique. Le lancement " +"Web UI, sélectionnez vLLM, SGLang ou MLX, activez le placement par réplique " +"et choisissez Prefill ou Decode pour chaque réplique. Le lancement " "communautaire utilise Xavier par défaut lorsque des rôles P/D sont définis." #: source/user_guide/pd_separation.rst:57 @@ -484,8 +484,9 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier stocke des pages CPU immuables dans un actor partagé du " "supervisor et les transfère par RPC actor. ``xavier_cache_bytes`` limite ce " @@ -496,8 +497,9 @@ msgstr "" "premier token de réponse. Les requêtes avec ou sans streaming sont prises en" " charge. Les transferts actifs réservent la capacité du cache ; une capacité" " insuffisante, des métadonnées incompatibles ou un échec de transfert " -"provoquent une erreur. Les transferts non récupérés expirent après cinq " -"minutes." +"provoquent une erreur. Le délai de transfert de cinq minutes commence " +"lorsque Prefill réserve la capacité, avant de calculer le préfixe, et inclut" +" le temps de prefill et de transfert." #: user_guide/pd_separation.rst msgid "" @@ -513,3 +515,24 @@ msgstr "" "répliques sur un même Mac partagent son GPU Metal ; mesurez votre charge " "avant d’attendre un gain de débit. Le P/D entre moteurs, notamment de NVIDIA" " vers Mac, n’est pas encore pris en charge." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"Chaque page FP16 de 64 tokens utilise ``2 * 64 * num_layers * num_kv_heads *" +" head_dim * 2`` octets (K et V, avec deux octets par valeur). Par exemple, " +"36 couches, 8 têtes KV et une dimension de tête de 128 nécessitent 9 MiB par" +" page ; 512 MiB contiennent 56 pages, soit 3584 tokens de préfixe. " +"Arrondissez chaque préfixe à une page entière et dimensionnez " +"``xavier_cache_bytes`` pour les pages distinctes réservées par toutes les " +"requêtes P/D simultanées. Les requêtes ordinaires avec cache partagé " +"omettent la publication lorsque leur préfixe dépasse la capacité du cache ou" +" la limite du protocole de 4096 pages." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po index 2c16f8d1b6..a34d329394 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po @@ -59,16 +59,16 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "Lo stesso ``replica_config`` è accettato da ``POST /v1/models``, dal client " "Python asincrono e dall’opzione JSON ``--replica_config`` della CLI. Nella " -"Web UI, selezionare vLLM o SGLang, attivare il posizionamento per replica e " -"scegliere Prefill o Decode per ciascuna replica. Il percorso di avvio della " -"versione community usa Xavier per impostazione predefinita quando sono " -"presenti ruoli P/D." +"Web UI, selezionare vLLM, SGLang o MLX, attivare il posizionamento per " +"replica e scegliere Prefill o Decode per ciascuna replica. Il percorso di " +"avvio della versione community usa Xavier per impostazione predefinita " +"quando sono presenti ruoli P/D." #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -480,8 +480,9 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier conserva pagine CPU immutabili in un actor condiviso sul " "supervisor e le trasferisce tramite RPC degli actor. ``xavier_cache_bytes`` " @@ -491,8 +492,9 @@ msgstr "" "prefisso, calcola l’ultimo token e campiona il primo token di risposta. Sono" " supportate richieste con e senza streaming. I trasferimenti attivi " "riservano capacità della cache; capacità insufficiente, metadati " -"incompatibili o errori di trasferimento generano un errore. I trasferimenti " -"non ritirati scadono dopo cinque minuti." +"incompatibili o errori di trasferimento generano un errore. Il termine di " +"cinque minuti inizia quando Prefill riserva capacità, prima di calcolare il " +"prefisso, e comprende il tempo di prefill e trasferimento." #: user_guide/pd_separation.rst msgid "" @@ -508,3 +510,24 @@ msgstr "" "repliche sullo stesso Mac condividono la GPU Metal; misurare il carico prima" " di aspettarsi un aumento del throughput. Il P/D tra motori, incluso il " "trasferimento da NVIDIA a Mac, non è ancora supportato." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"Ogni pagina FP16 di 64 token usa ``2 * 64 * num_layers * num_kv_heads * " +"head_dim * 2`` byte (K e V, con due byte per valore). Ad esempio, 36 " +"livelli, 8 teste KV e una dimensione della testa di 128 richiedono 9 MiB per" +" pagina; 512 MiB contengono 56 pagine, ovvero 3584 token di prefisso. " +"Arrotondare ogni prefisso a una pagina intera e dimensionare " +"``xavier_cache_bytes`` per le pagine distinte riservate da tutte le " +"richieste P/D simultanee. Le richieste ordinarie con cache condivisa saltano" +" la pubblicazione quando il prefisso supera la capacità della cache o il " +"limite del protocollo di 4096 pagine." diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po index 728ee91889..dfa74ec4bd 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po @@ -57,12 +57,12 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "同じ ``replica_config`` を ``POST /v1/models``、非同期 Python クライアント、CLI の " -"``--replica_config`` JSON オプションで使用できます。Web UI では vLLM または SGLang " +"``--replica_config`` JSON オプションで使用できます。Web UI では vLLM、SGLang または MLX " "を選択し、レプリカごとの配置を有効にして、各レプリカに Prefill または Decode を指定します。P/D " "ロールを指定すると、コミュニティ版の起動処理はデフォルトで Xavier を使用します。" @@ -436,15 +436,16 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier は supervisor の共有 actor に不変の CPU ページを保存し、actor RPC " "で転送します。``xavier_cache_bytes`` はデプロイ全体のキャッシュ容量を制限します（既定値 512 " "MiB）。モデルの重みと各レプリカの Metal KV 割り当てには追加メモリが必要です。Prefill " "はプロンプトの最後のトークン以外を計算します。Decode " -"はプレフィックスをインポートし、最後のトークンを計算して最初の応答トークンをサンプリングします。ストリーミングと非ストリーミングに対応します。実行中の引き継ぎはキャッシュ容量を予約し、容量不足、メタデータ不一致、転送失敗はエラーになります。未取得の引き継ぎは" -" 5 分で期限切れになります。" +"はプレフィックスをインポートし、最後のトークンを計算して最初の応答トークンをサンプリングします。ストリーミングと非ストリーミングに対応します。実行中の引き継ぎはキャッシュ容量を予約し、容量不足、メタデータ不一致、転送失敗はエラーになります。引き継ぎの" +" 5 分間の期限は、Prefill が容量を予約した時点、プレフィックスの計算前から始まり、prefill と転送の時間を含みます。" #: user_guide/pd_separation.rst msgid "" @@ -458,3 +459,21 @@ msgstr "" "actor アドレスに接続できる必要があります。Worker 再起動後はデプロイを再起動してください。同じ Mac 上の 2 レプリカは Metal " "GPU を共有するため、スループット改善は実際の負荷で測定してください。NVIDIA から Mac への引き継ぎを含む異なるエンジン間の P/D " "は未対応です。" + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"64 トークンの FP16 ページは ``2 * 64 * num_layers * num_kv_heads * head_dim * 2`` " +"バイトを使用します（K と V、各値は 2 バイト）。例えば 36 層、8 KV ヘッド、ヘッド次元 128 では 1 ページに 9 MiB " +"が必要で、512 MiB は 56 ページ、つまり 3584 プレフィックストークンを保持できます。各プレフィックスをページ単位で切り上げ、同時 P/D" +" リクエスト全体が予約する異なるページ数に合わせて ``xavier_cache_bytes`` " +"を設定してください。通常の共有キャッシュリクエストは、プレフィックスがキャッシュ容量またはプロトコルの 4096 " +"ページ制限を超える場合、公開を省略します。" diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po index 350484c175..c3a1c9ca08 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po @@ -57,14 +57,14 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "동일한 ``replica_config`` 를 ``POST /v1/models``, 비동기 Python 클라이언트 및 CLI의 " -"``--replica_config`` JSON 옵션에서 사용할 수 있습니다. Web UI에서 vLLM 또는 SGLang을 선택하고 " -"복제본별 배치를 활성화한 다음 각 복제본에 Prefill 또는 Decode를 지정합니다. P/D 역할을 지정하면 커뮤니티 버전의 실행 " -"경로는 기본적으로 Xavier를 사용합니다." +"``--replica_config`` JSON 옵션에서 사용할 수 있습니다. Web UI에서 vLLM, SGLang 또는 MLX을 " +"선택하고 복제본별 배치를 활성화한 다음 각 복제본에 Prefill 또는 Decode를 지정합니다. P/D 역할을 지정하면 커뮤니티 버전의" +" 실행 경로는 기본적으로 Xavier를 사용합니다." #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -435,14 +435,16 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier는 supervisor의 공유 actor에 불변 CPU 페이지를 저장하고 actor RPC로 전송합니다. " "``xavier_cache_bytes`` 는 배포 캐시 용량을 제한합니다(기본 512 MiB). 모델 가중치와 각 복제본의 Metal " "KV 할당에는 추가 메모리가 필요합니다. Prefill은 마지막 프롬프트 토큰을 제외한 모든 토큰을 계산합니다. Decode는 접두사를 " "가져와 마지막 토큰을 계산하고 첫 응답 토큰을 샘플링합니다. 스트리밍 및 비스트리밍 요청을 지원합니다. 활성 인계는 캐시 용량을 예약하며" -" 용량 부족, 메타데이터 불일치 또는 전송 실패는 오류를 발생시킵니다. 수령하지 않은 인계는 5분 후 만료됩니다." +" 용량 부족, 메타데이터 불일치 또는 전송 실패는 오류를 발생시킵니다. 5분 인계 기한은 Prefill이 용량을 예약할 때, 접두사 계산" +" 전에 시작되며 prefill 및 전송 시간을 포함합니다." #: user_guide/pd_separation.rst msgid "" @@ -456,3 +458,20 @@ msgstr "" "actor 주소에 접근할 수 있어야 합니다. Worker 재시작 후 배포를 다시 시작하세요. 같은 Mac의 두 복제본은 Metal " "GPU를 공유하므로 처리량 향상을 기대하기 전에 워크로드를 측정하세요. NVIDIA에서 Mac으로의 인계를 포함한 엔진 간 P/D는 아직" " 지원하지 않습니다." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"64토큰 FP16 페이지마다 ``2 * 64 * num_layers * num_kv_heads * head_dim * 2`` 바이트를 " +"사용합니다(K와 V, 값마다 2바이트). 예를 들어 36개 레이어, 8개 KV 헤드, 헤드 차원 128이면 페이지당 9 MiB가 필요하며" +" 512 MiB는 56페이지, 즉 접두사 3584토큰을 저장합니다. 각 접두사를 전체 페이지 단위로 올림하고 모든 동시 P/D 요청이 " +"예약하는 서로 다른 페이지에 맞게 ``xavier_cache_bytes`` 를 설정하십시오. 일반 공유 캐시 요청은 접두사가 캐시 용량 " +"또는 프로토콜의 4096페이지 한도를 초과하면 게시를 건너뜁니다." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po index b42ce64e62..f44947531c 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po @@ -59,15 +59,15 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "O mesmo ``replica_config`` é aceito por ``POST /v1/models``, pelo cliente " "Python assíncrono e pela opção JSON ``--replica_config`` da CLI. Na Web UI, " -"selecione vLLM ou SGLang, habilite o posicionamento por réplica e escolha " -"Prefill ou Decode para cada réplica. O fluxo de inicialização da edição " -"comunitária usa Xavier por padrão quando há funções P/D." +"selecione vLLM, SGLang ou MLX, habilite o posicionamento por réplica e " +"escolha Prefill ou Decode para cada réplica. O fluxo de inicialização da " +"edição comunitária usa Xavier por padrão quando há funções P/D." #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -475,8 +475,9 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier armazena páginas CPU imutáveis em um actor compartilhado no " "supervisor e as transfere por RPC de actors. ``xavier_cache_bytes`` limita " @@ -486,8 +487,9 @@ msgstr "" "calcula o último token e amostra o primeiro token de resposta. Solicitações " "com e sem streaming são suportadas. Transferências ativas reservam " "capacidade do cache; capacidade insuficiente, metadados incompatíveis ou " -"falha de transferência geram um erro. Transferências não recebidas expiram " -"após cinco minutos." +"falha de transferência geram um erro. O prazo de cinco minutos começa quando" +" Prefill reserva capacidade, antes de calcular o prefixo, e inclui o tempo " +"de prefill e transferência." #: user_guide/pd_separation.rst msgid "" @@ -502,3 +504,23 @@ msgstr "" "deployment após reiniciar um worker. Duas réplicas no mesmo Mac compartilham" " a GPU Metal; meça sua carga antes de esperar ganho de throughput. P/D entre" " engines, incluindo transferência de NVIDIA para Mac, ainda não é suportado." + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"Cada página FP16 de 64 tokens usa ``2 * 64 * num_layers * num_kv_heads * " +"head_dim * 2`` bytes (K e V, com dois bytes por valor). Por exemplo, 36 " +"camadas, 8 cabeças KV e dimensão de cabeça 128 exigem 9 MiB por página; 512 " +"MiB comportam 56 páginas, ou 3584 tokens de prefixo. Arredonde cada prefixo " +"para uma página inteira e dimensione ``xavier_cache_bytes`` para as páginas " +"distintas reservadas por todas as solicitações P/D simultâneas. Solicitações" +" comuns de cache compartilhado pulam a publicação quando o prefixo excede a " +"capacidade do cache ou o limite de protocolo de 4096 páginas." diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po index 37c0f3cab4..8ce58e505a 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po @@ -59,13 +59,13 @@ msgstr "示例在 GPU 0 上启动一个 Prefill 副本，在 GPU 1 上启动一 msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "``POST /v1/models``、异步 Python 客户端和 CLI 的 ``--replica_config`` JSON 选项均接受相同的 " -"``replica_config``。在 Web UI 中，选择 vLLM 或 SGLang，启用逐副本放置，并为每个副本选择 Prefill 或 " -"Decode。出现 P/D 角色时，社区版启动路径默认使用 Xavier。" +"``replica_config``。在 Web UI 中，选择 vLLM、SGLang 或 MLX，启用逐副本放置，并为每个副本选择 Prefill " +"或 Decode。出现 P/D 角色时，社区版启动路径默认使用 Xavier。" #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -427,13 +427,15 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier 在 supervisor 上的共享 actor 中保存不可变 CPU 页，通过 actor RPC " "传输。``xavier_cache_bytes`` 限制部署缓存容量，默认 512 MiB；模型权重和各副本的 Metal KV " "分配需要额外内存。Prefill 副本计算除最后一个之外的全部提示词 token。Decode 导入前缀，计算最后一个 token 并采样首个响应 " -"token。支持流式和非流式请求。活跃交接会预留缓存容量；容量不足、元数据不兼容或传输失败会报错。未领取的交接在五分钟后过期。" +"token。支持流式和非流式请求。活跃交接会预留缓存容量；容量不足、元数据不兼容或传输失败会报错。交接的五分钟期限从 Prefill " +"预留容量时开始，早于前缀计算，包含 prefill 和传输时间。" #: user_guide/pd_separation.rst msgid "" @@ -446,3 +448,19 @@ msgstr "" "MLX 使用默认的 ``xavier`` 传输，不支持原生 NIXL。所有 worker 必须能访问 supervisor 的 actor " "地址。Worker 重启后需要重新启动部署。同一台 Mac 上的两个副本共享 Metal GPU，应先测量工作负载，再评估吞吐收益。目前不支持跨引擎 " "P/D，包括 NVIDIA 到 Mac 的交接。" + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"每个 64 token 的 FP16 页占用 ``2 * 64 * num_layers * num_kv_heads * head_dim * 2``" +" 字节（K 和 V，每个值占两字节）。例如，36 层、8 个 KV 头、头维度为 128 时，每页需要 9 MiB；512 MiB 可保存 56 页，即" +" 3584 个前缀 token。将每个前缀向上取整到整页，并按所有并发 P/D 请求预留的不同页总量配置 " +"``xavier_cache_bytes``。普通共享缓存请求的前缀超过缓存容量或 4096 页协议上限时，会跳过发布。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po index b1e5961168..c5bc3f835a 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po @@ -57,13 +57,13 @@ msgstr "" msgid "" "The same ``replica_config`` is accepted by ``POST /v1/models``, the async " "Python client, and the CLI ``--replica_config`` JSON option. In the Web UI, " -"select vLLM or SGLang, enable per-replica placement, and choose Prefill or " -"Decode for each replica. The community launch path defaults to Xavier when " -"P/D roles appear." +"select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill" +" or Decode for each replica. The community launch path defaults to Xavier " +"when P/D roles appear." msgstr "" "``POST /v1/models``、非同步 Python 用戶端及 CLI 的 ``--replica_config`` JSON 選項均接受相同的" -" ``replica_config``。在 Web UI 中，選擇 vLLM 或 SGLang，啟用逐副本配置，並為每個副本選擇 Prefill 或 " -"Decode。出現 P/D 角色時，社群版啟動流程預設使用 Xavier。" +" ``replica_config``。在 Web UI 中，選擇 vLLM、SGLang 或 MLX，啟用逐副本配置，並為每個副本選擇 Prefill" +" 或 Decode。出現 P/D 角色時，社群版啟動流程預設使用 Xavier。" #: source/user_guide/pd_separation.rst:57 msgid "CLI launch" @@ -425,13 +425,15 @@ msgid "" " the last prompt token. Decode imports the prefix, evaluates that last token" " and samples the first response token. Streaming and non-streaming requests " "are supported. Active handoffs reserve cache capacity; insufficient " -"capacity, incompatible metadata or transfer failure raises an error. " -"Unclaimed handoffs expire after five minutes." +"capacity, incompatible metadata or transfer failure raises an error. The " +"five-minute handoff deadline starts when prefill reserves capacity, before " +"computing the prefix, and includes prefill and transfer time." msgstr "" "MLX Xavier 在 supervisor 的共用 actor 中儲存不可變 CPU 頁面，透過 actor RPC " "傳輸。``xavier_cache_bytes`` 限制部署快取容量，預設 512 MiB；模型權重和各副本的 Metal KV " "配置需要額外記憶體。Prefill 副本計算除最後一個以外的全部提示 token。Decode 匯入前綴，計算最後一個 token 並取樣首個回應 " -"token。支援串流和非串流請求。活躍交接會保留快取容量；容量不足、中繼資料不相容或傳輸失敗會報錯。未領取的交接在五分鐘後到期。" +"token。支援串流和非串流請求。活躍交接會保留快取容量；容量不足、中繼資料不相容或傳輸失敗會報錯。交接的五分鐘期限從 Prefill " +"保留容量時開始，早於前綴計算，包含 prefill 與傳輸時間。" #: user_guide/pd_separation.rst msgid "" @@ -444,3 +446,19 @@ msgstr "" "MLX 使用預設的 ``xavier`` 傳輸，不支援原生 NIXL。所有 worker 必須能存取 supervisor 的 actor " "位址。Worker 重新啟動後需要重新啟動部署。同一台 Mac 上的兩個副本共用 Metal GPU，應先測量工作負載，再評估吞吐收益。目前不支援跨引擎" " P/D，包括 NVIDIA 到 Mac 的交接。" + +#: user_guide/pd_separation.rst:102 +msgid "" +"Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim" +" * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 " +"KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds " +"56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and " +"size ``xavier_cache_bytes`` for the distinct pages reserved by all " +"concurrent P/D requests. Ordinary shared-cache requests skip publication " +"when their prefix exceeds the cache capacity or the 4096-page protocol " +"limit." +msgstr "" +"每個 64 token 的 FP16 頁占用 ``2 * 64 * num_layers * num_kv_heads * head_dim * 2``" +" 位元組（K 與 V，每個值占兩個位元組）。例如，36 層、8 個 KV 頭、頭維度為 128 時，每頁需要 9 MiB；512 MiB 可儲存 56 " +"頁，即 3584 個前綴 token。將每個前綴向上取整到整頁，並按所有並行 P/D 請求保留的不同頁總量設定 " +"``xavier_cache_bytes``。一般共享快取請求的前綴超過快取容量或 4096 頁協定上限時，會跳過發佈。" diff --git a/doc/source/user_guide/pd_separation.rst b/doc/source/user_guide/pd_separation.rst index 59e44236ed..4e3e2f8efa 100644 --- a/doc/source/user_guide/pd_separation.rst +++ b/doc/source/user_guide/pd_separation.rst @@ -46,7 +46,7 @@ configuration. ) The same ``replica_config`` is accepted by ``POST /v1/models``, the async Python -client, and the CLI ``--replica_config`` JSON option. In the Web UI, select vLLM or SGLang, +client, and the CLI ``--replica_config`` JSON option. In the Web UI, select vLLM, SGLang or MLX, enable per-replica placement, and choose Prefill or Decode for each replica. The community launch path defaults to Xavier when P/D roles appear. @@ -97,7 +97,9 @@ Use ``model_engine="MLX"``, ``model_format="mlx"`` and ``quantization="none"`` w ], ) -MLX Xavier stores immutable CPU pages in a shared actor on the supervisor and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this deployment cache (default: 512 MiB); model weights and per-replica Metal KV allocations consume additional memory. The prefill replica evaluates all but the last prompt token. Decode imports the prefix, evaluates that last token and samples the first response token. Streaming and non-streaming requests are supported. Active handoffs reserve cache capacity; insufficient capacity, incompatible metadata or transfer failure raises an error. Unclaimed handoffs expire after five minutes. +MLX Xavier stores immutable CPU pages in a shared actor on the supervisor and transfers them through actor RPC. ``xavier_cache_bytes`` bounds this deployment cache (default: 512 MiB); model weights and per-replica Metal KV allocations consume additional memory. The prefill replica evaluates all but the last prompt token. Decode imports the prefix, evaluates that last token and samples the first response token. Streaming and non-streaming requests are supported. Active handoffs reserve cache capacity; insufficient capacity, incompatible metadata or transfer failure raises an error. The five-minute handoff deadline starts when prefill reserves capacity, before computing the prefix, and includes prefill and transfer time. + +Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds 56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and size ``xavier_cache_bytes`` for the distinct pages reserved by all concurrent P/D requests. Ordinary shared-cache requests skip publication when their prefix exceeds the cache capacity or the 4096-page protocol limit. Use the default ``xavier`` transport for MLX; native NIXL is unsupported. All workers must reach the supervisor's actor address. Relaunch after a worker restart. Two replicas on one Mac share its Metal GPU; measure your workload before expecting a throughput benefit. Cross-engine P/D, including NVIDIA-to-Mac handoff, is not yet supported. diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index 703b7cf44e..157bb3bb7f 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3126,10 +3126,8 @@ async def launch_builtin_model( and (model_engine or "").lower() == "mlx" ) if (requested_xavier or pd_enabled) and (model_engine or "").lower() == "mlx": - if transport_backend != "xavier" or n_worker != 1: - raise ValueError( - "MLX Xavier requires the xavier transport and one worker per replica" - ) + if transport_backend != "xavier": + raise ValueError("MLX Xavier requires the xavier transport") if ( model_type not in (None, "LLM") or model_format != "mlx" diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index 18d51d22a4..406b883f16 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -135,7 +135,7 @@ async def test_mlx_pd_recovery_keeps_bytes_cache_and_registers_replacement(role) cache = await xo.create_actor( XavierBytesCacheActor, address=pool.external_address, uid="bytes-cache" ) - namespace = await cache.configure(contract().to_dict()) + namespace = (await cache.configure(contract().to_dict()))["namespace"] worker = MagicMock() supervisor = AsyncMock() worker.get_supervisor_ref = AsyncMock(return_value=supervisor) @@ -176,7 +176,6 @@ async def test_mlx_failed_launch_cleans_bytes_actor(launch_runtime): "options", [ {"transfer_backend_type": "nixl"}, - {"n_worker": 2}, {"model_format": "pytorch"}, {"quantization": "4bit"}, {"xavier_gpu_cache_bytes": 1024}, diff --git a/xinference/model/llm/mlx/core.py b/xinference/model/llm/mlx/core.py index 2afe36b676..d2268e646d 100644 --- a/xinference/model/llm/mlx/core.py +++ b/xinference/model/llm/mlx/core.py @@ -1778,6 +1778,8 @@ def __init__( model_config, peft_model, ) + if self._xavier_config is not None: + raise ValueError("MLX Xavier does not support vision models") self._mlx_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None self._draft_model: Optional[Any] = None self._draft_kind: Optional[str] = None diff --git a/xinference/model/llm/mlx/tests/test_xavier.py b/xinference/model/llm/mlx/tests/test_xavier.py index 1c228ef8d0..10e02805ca 100644 --- a/xinference/model/llm/mlx/tests/test_xavier.py +++ b/xinference/model/llm/mlx/tests/test_xavier.py @@ -59,6 +59,24 @@ def model(model_type="qwen2"): return value +@pytest.mark.parametrize("role", [None, "prefill", "decode"]) +def test_vision_model_rejects_xavier_before_loading(role): + from ..core import MLXVisionModel + + family = SimpleNamespace( + model_specs=[SimpleNamespace(quantization="bf16", model_revision=None)], + model_ability=["chat", "vision"], + ) + with pytest.raises(ValueError, match="does not support vision"): + MLXVisionModel( + "vision-rep0", + family, + "unused", + {"_xavier_cache_config": dict(address="address", uid="cache", role=role)}, + ) + assert MLXVisionModel("vision-rep0", family, "unused", {})._xavier_config is None + + @pytest.mark.asyncio async def test_pd_metadata_is_checked_before_reading(): client = MLXXavierCache( @@ -206,6 +224,9 @@ async def test_publication_failure_does_not_drop_batch_results(monkeypatch, capl @pytest.mark.asyncio async def test_cancelled_flush_keeps_other_requests_writes_independent(): client = MLXXavierCache(contract(), {}) + client._ref = SimpleNamespace( + configure=AsyncMock(return_value={"capacity_pages": 4096, "max_keys": 4096}) + ) client.encode = MagicMock(return_value=[b"page"]) gates = {client.keys([i])[0]: asyncio.Event() for i in (1, 2)} @@ -227,6 +248,29 @@ async def put(method, namespace, keys, pages, start): await asyncio.wait_for(client.flush([first]), 1) +@pytest.mark.asyncio +@pytest.mark.parametrize("capacity,length", [(2, 129), (4097, 4096 * 64 + 1)]) +async def test_oversized_shared_prompt_skips_encoding_and_publication(capacity, length): + c = contract() + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, + capacity * c.layer_nbytes * c.num_layers, + address=pool.external_address, + ) + client = MLXXavierCache(c, dict(address=ref.address, uid=ref.uid)) + client.encode = MagicMock(side_effect=AssertionError("must not encode")) + client.keys = MagicMock(side_effect=AssertionError("must not build keys")) + for _ in range(2): + await client.publish([], [1] * length) + client.encode.assert_not_called() + client.keys.assert_not_called() + assert client._capacity_pages == min(capacity, 4096) + stats = await ref.get_stats() + assert stats["stored_pages"] == 0 and stats["pages"] == 0 + + @pytest.mark.asyncio @pytest.mark.parametrize("decode_error", [False, True]) async def test_failed_release_keeps_decode_result_or_original_error( @@ -343,6 +387,9 @@ async def test_publish_encodes_only_pages_beyond_remote_prefix(cached): mx.eval(m(mx.array(tokens)[None], cache=cache)) client = MLXXavierCache(contract(), {}) complete = client.encode(cache, tokens) + client._ref = SimpleNamespace( + configure=AsyncMock(return_value={"capacity_pages": 4096, "max_keys": 4096}) + ) client._call = AsyncMock() await client.publish(cache, tokens, cached_tokens=cached) client._call.assert_awaited_once_with( @@ -490,6 +537,41 @@ async def test_real_pd_import_skips_prefill_and_matches_logits(): assert (await ref.get_stats())["active_handoffs"] == 0 +@metal +@pytest.mark.asyncio +@pytest.mark.parametrize("cached", [64, 128]) +async def test_partial_hit_prefill_computes_and_publishes_only_missing_suffix(cached): + import mlx.core as mx + from mlx_lm.models.cache import make_prompt_cache + + pool = await xo.create_actor_pool("127.0.0.1", n_process=0) + async with pool: + ref = await xo.create_actor( + XavierBytesCacheActor, address=pool.external_address, uid="mlx-cache" + ) + cfg = dict(address=ref.address, uid=ref.uid) + p = MLXXavierCache(contract(), dict(cfg, role="prefill")) + d = MLXXavierCache(contract(), dict(cfg, role="decode")) + tokens = [i % 128 for i in range(130)] + m = model() + seed = await p.prefill(m, tokens[: cached + 1], prefix_length=24) + await d.fetch(tokens[: cached + 1], seed) + p.imported_tokens = 0 + p.encode = MagicMock(wraps=p.encode) + transfer = await p.prefill(m, tokens, prefix_length=24) + assert p.imported_tokens == cached + p.encode.assert_called_once() + assert p.encode.call_args.args[1:] == (tokens[:-1], cached) + cache, reused = await d.fetch(tokens, transfer) + assert reused == len(tokens) - 1 + actual = m(mx.array(tokens[reused:])[None], cache=cache)[:, -1] + expected = m(mx.array(tokens)[None], cache=make_prompt_cache(m))[:, -1] + mx.eval(actual, expected) + assert bool(mx.array_equal(mx.argmax(actual, -1), mx.argmax(expected, -1))) + assert float(mx.max(mx.abs(actual.astype(mx.float32) - expected))) < 0.02 + assert (await ref.get_stats())["active_handoffs"] == 0 + + @metal @pytest.mark.asyncio @pytest.mark.parametrize("release_fails", [False, True]) diff --git a/xinference/model/llm/mlx/xavier.py b/xinference/model/llm/mlx/xavier.py index 4acfe328b8..9f3ae16a28 100644 --- a/xinference/model/llm/mlx/xavier.py +++ b/xinference/model/llm/mlx/xavier.py @@ -113,6 +113,7 @@ def __init__(self, contract: KVCacheContract, config: dict): self.role = config.get("role") self._ref = None self._configured = False + self._capacity_pages = 0 self._writes: set = set() self.imported_tokens = 0 @@ -122,7 +123,8 @@ async def _ensure_configured(self): address=self.config["address"], uid=self.config["uid"] ) if not self._configured: - await self._ref.configure(self.contract.to_dict()) + metadata = await self._ref.configure(self.contract.to_dict()) + self._capacity_pages = min(metadata["capacity_pages"], metadata["max_keys"]) self._configured = True async def initialize(self): @@ -268,12 +270,18 @@ async def _release_handoff(self, ticket): logger.warning("MLX Xavier handoff release failed", exc_info=True) def publish(self, cache, tokens, cached_tokens=0): - start = cached_tokens // self.contract.block_size * self.contract.block_size - keys = self.keys(tokens) - pages = self.encode(cache, tokens, start) - async def write(): try: + await self.initialize() + if ( + len(tokens) + self.contract.block_size - 1 + ) // self.contract.block_size > self._capacity_pages: + return + start = ( + cached_tokens // self.contract.block_size * self.contract.block_size + ) + keys = self.keys(tokens) + pages = self.encode(cache, tokens, start) await self._call( "put", self.contract.fingerprint, @@ -289,11 +297,10 @@ async def write(): task.add_done_callback(self._writes.discard) return task - async def flush(self, writes=None): - tasks = list(self._writes) if writes is None else writes - if tasks: + async def flush(self, writes): + if writes: await asyncio.gather( - *(asyncio.shield(task) for task in tasks), return_exceptions=True + *(asyncio.shield(task) for task in writes), return_exceptions=True ) async def prefill(self, model, tokens, prefix_length=None): diff --git a/xinference/model/llm/xavier/backends/bytes/storage.py b/xinference/model/llm/xavier/backends/bytes/storage.py index b9c986c5f8..44837ee2dc 100644 --- a/xinference/model/llm/xavier/backends/bytes/storage.py +++ b/xinference/model/llm/xavier/backends/bytes/storage.py @@ -38,7 +38,11 @@ def configure(self, metadata): self._contract = contract self._page_bytes = page_bytes self._capacity = self.capacity_bytes // page_bytes - return contract.fingerprint + return { + "namespace": contract.fingerprint, + "capacity_pages": self._capacity, + "max_keys": 4096, + } def _validate(self, namespace, keys): if self._contract is None or namespace != self._contract.fingerprint: @@ -80,6 +84,11 @@ def put(self, namespace, keys, pages, start=0): if len(keys) > self._capacity: return [False] * len(keys) pinned = self._pinned() + # Refresh an aged prefix before inserting any suffix pages. Otherwise + # insertion can evict this chain's own head before the final refresh. + for key in reversed(keys[:start]): + if key in self._pages: + self._pages.move_to_end(key) for key, page in reversed(list(zip(keys[start:], pages))): if key not in self._pages: if len(self._pages) >= self._capacity: diff --git a/xinference/model/llm/xavier/tests/test_bytes_storage.py b/xinference/model/llm/xavier/tests/test_bytes_storage.py index 6a23baa9f0..075af55834 100644 --- a/xinference/model/llm/xavier/tests/test_bytes_storage.py +++ b/xinference/model/llm/xavier/tests/test_bytes_storage.py @@ -20,7 +20,9 @@ def contract(): def store(capacity=2): c = contract() actor = XavierBytesCacheActor(capacity * c.layer_nbytes * c.num_layers) - namespace = actor.configure(c.to_dict()) + metadata = actor.configure(c.to_dict()) + assert metadata["capacity_pages"] == capacity and metadata["max_keys"] == 4096 + namespace = metadata["namespace"] page = bytes(c.layer_nbytes * c.num_layers) return actor, namespace, page @@ -79,6 +81,31 @@ def test_suffix_publication_keeps_existing_chain_heads_newer_than_tails(): actor.put(namespace, keys[:4], [page], start=2) +def test_suffix_publication_refreshes_aged_prefix_before_eviction(): + actor, namespace, page = store(4) + keys = [str(i) * 64 for i in range(1, 7)] + actor.put(namespace, keys[:2], [page] * 2) + actor.put(namespace, keys[4:], [page] * 2) + assert actor.put(namespace, keys[:4], [page] * 2, start=2) == [True] * 4 + assert actor.get(namespace, keys[:4]) == [page] * 4 + assert actor.get(namespace, keys[4:]) == [] + assert actor.get_stats()["evicted_pages"] == 2 + + +def test_prepare_handoff_evicts_unpinned_pages_and_preserves_active_prefix(): + actor, namespace, page = store(3) + keys = [str(i) * 64 for i in range(1, 6)] + actor.put(namespace, keys[:3], [page] * 3) + existing = actor.prepare_handoff(namespace, keys[:1]) + incoming = actor.prepare_handoff(namespace, keys[3:]) + assert actor.get_stats()["evicted_pages"] == 2 + assert actor.get_handoff(namespace, keys[:1], existing) == [page] + assert not actor.handoff_ready(namespace, keys[3:], incoming) + assert actor.put(namespace, keys[3:], [page] * 2) == [True] * 2 + assert actor.get_handoff(namespace, keys[3:], incoming) == [page] * 2 + assert actor.get_stats()["pages"] == 3 + + def test_leases_pin_absent_pages_and_handoff_is_consumed_once(): actor, namespace, page = store() keys = ["1" * 64, "2" * 64]