From 6d4d70a07642092875c37bc55e5b1e519de06253 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 00:41:38 +0800 Subject: [PATCH 1/8] feat: add Xavier PD handoff across vLLM and SGLang --- benchmark/benchmark_cross_engine_pd.py | 377 ++++++++++++++++++ .../LC_MESSAGES/user_guide/pd_separation.po | 89 ++++- .../LC_MESSAGES/user_guide/pd_separation.po | 89 ++++- .../LC_MESSAGES/user_guide/pd_separation.po | 91 ++++- .../LC_MESSAGES/user_guide/pd_separation.po | 89 ++++- .../LC_MESSAGES/user_guide/pd_separation.po | 82 +++- .../LC_MESSAGES/user_guide/pd_separation.po | 82 +++- .../LC_MESSAGES/user_guide/pd_separation.po | 88 +++- .../LC_MESSAGES/user_guide/pd_separation.po | 79 +++- .../LC_MESSAGES/user_guide/pd_separation.po | 79 +++- doc/source/user_guide/pd_separation.rst | 41 +- xinference/core/pd_model.py | 45 ++- xinference/core/replica_config.py | 15 +- xinference/core/supervisor.py | 64 ++- xinference/core/tests/test_pd_launch.py | 37 ++ xinference/core/tests/test_pd_model.py | 79 +++- xinference/model/llm/sglang/xavier/config.py | 6 + .../model/llm/sglang/xavier/directory.py | 23 +- xinference/model/llm/sglang/xavier/gpu.py | 20 +- xinference/model/llm/sglang/xavier/pd.py | 19 +- xinference/model/llm/vllm/core.py | 16 +- .../model/llm/vllm/xavier/cross_engine.py | 276 +++++++++++++ xinference/model/llm/vllm/xavier/engine.py | 15 +- .../llm/vllm/xavier/test/test_cross_engine.py | 102 +++++ .../model/llm/xavier/backends/torch/pd.py | 353 ++++++++++++++++ xinference/model/llm/xavier/pd_contract.py | 76 ++++ .../llm/xavier/tests/test_pd_cross_engine.py | 108 +++++ .../xavier/tests/test_pd_cross_engine_gpu.py | 113 ++++++ 28 files changed, 2453 insertions(+), 100 deletions(-) create mode 100644 benchmark/benchmark_cross_engine_pd.py create mode 100644 xinference/model/llm/vllm/xavier/cross_engine.py create mode 100644 xinference/model/llm/vllm/xavier/test/test_cross_engine.py create mode 100644 xinference/model/llm/xavier/backends/torch/pd.py create mode 100644 xinference/model/llm/xavier/pd_contract.py create mode 100644 xinference/model/llm/xavier/tests/test_pd_cross_engine.py create mode 100644 xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py diff --git a/benchmark/benchmark_cross_engine_pd.py b/benchmark/benchmark_cross_engine_pd.py new file mode 100644 index 0000000000..5e9f967650 --- /dev/null +++ b/benchmark/benchmark_cross_engine_pd.py @@ -0,0 +1,377 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""Controlled monolithic/native/heterogeneous PD comparisons on two CUDA GPUs.""" + +import argparse +import asyncio +import json +import os +import subprocess +import sys +import time +import uuid +from importlib.metadata import version +from pathlib import Path + +import requests +import xoscar as xo +from benchmark_pd import measure, summarize +from benchmark_sglang_pd import GPUIsolation, stop_server +from openai import AsyncOpenAI +from xoscar.utils import get_next_port + +from xinference.client import Client + + +def engine_options(engine, role, args): + if engine == "vllm": + return dict( + dtype="float16", + max_model_len=8192, + block_size=64, + gpu_memory_utilization=0.3, + num_gpu_blocks_override=args.kv_tokens // 64, + max_num_seqs=32, + enforce_eager=True, + enable_prefix_caching=role != "decode", + ) + return dict( + dtype="float16", + context_length=8192, + page_size=64, + mem_fraction_static=0.3, + max_total_tokens=args.kv_tokens, + disable_cuda_graph=True, + stream_interval=1, + triton_attention_reduce_in_fp32=False, + ) + + +async def snapshot(worker, uid, heterogeneous): + ref = await xo.actor_ref(address=worker, uid="xavier-cache-" + uid) + directory = await ref.get_stats() + peers = {} + actor_uid = ( + "xavier-cross-engine-transfer" if heterogeneous else "sglang-xavier-transfer" + ) + for rank, address in directory["peers"].items(): + actor = await xo.actor_ref(address=address, uid=f"{actor_uid}-{rank}") + peers[rank] = await actor.get_stats() + return dict(directory=directory, peers=peers) + + +def bodies(): + return [ + dict( + messages=[ + dict( + role="user", + content="Water evaporates and condenses. " * repeats + + "Explain this process.", + ) + ], + temperature=0, + max_tokens=32, + stop=["__never_stop__"], + extra_body=dict(ignore_eos=True, repetition_penalty=1.0), + ) + for repeats in (1, 20, 100, 400) + ] + + +async def requests_and_measure(endpoint, uid, root, worker, mode, args): + api = AsyncOpenAI(base_url=endpoint + "/v1", api_key="unused", timeout=120) + results = dict(mode=mode, correctness=[], performance=[]) + try: + for body in bodies(): + regular = await api.chat.completions.create(model=uid, **body) + response = await api.chat.completions.create( + model=uid, **body, stream=True, stream_options=dict(include_usage=True) + ) + content, usage = [], None + async with response: + async for chunk in response: + if chunk.choices and chunk.choices[0].delta.content: + content.append(chunk.choices[0].delta.content) + if chunk.usage: + usage = chunk.usage.model_dump() + results["correctness"].append( + dict( + body=body, + text=regular.choices[0].message.content, + stream_text="".join(content), + usage=regular.usage.model_dump(), + stream_usage=usage, + ) + ) + assert regular.choices[0].message.content == "".join(content) + assert ( + regular.usage.completion_tokens == 32 + and usage["completion_tokens"] == 32 + ) + (root / "correctness.json").write_text( + json.dumps(results["correctness"], indent=2) + ) + for prompt in ( + "Explain the water cycle.", + "Evaporation and condensation. " * 100, + ): + body = dict( + prompt=prompt, + temperature=0, + max_tokens=32, + stop=["__never_stop__"], + extra_body=dict(ignore_eos=True), + ) + regular = await api.completions.create(model=uid, **body) + stream = await api.completions.create(model=uid, **body, stream=True) + pieces = [] + async with stream: + async for chunk in stream: + if chunk.choices: + pieces.append(chunk.choices[0].text) + assert regular.choices[0].text == "".join(pieces) + assert regular.usage.completion_tokens == 32 + results.setdefault("completion_correctness", []).append( + dict( + prompt=prompt, + text=regular.choices[0].text, + stream_text="".join(pieces), + ) + ) + for concurrency in args.concurrency: + # Repeated prefixes keep native P caching available in every mode. + requests_list = [bodies()[i % 4] for i in range(args.requests)] + records, elapsed, _ = await measure( + endpoint, uid, requests_list, concurrency, 1 + ) + assert all( + "error" not in row and row["output_tokens"] == 32 for row in records + ) + results["performance"].append( + dict( + concurrency=concurrency, + elapsed_s=elapsed, + summary=summarize(records, elapsed, 1.0, 0.05), + records=records, + ) + ) + if mode in ("vllm-sglang", "sglang-vllm"): + results["transfers"] = await snapshot(worker, uid, True) + assert results["transfers"]["directory"]["imported_tokens"] > 0 + assert all( + peer["cpu_batches"] == 0 + for peer in results["transfers"]["peers"].values() + ) + assert all( + peer["active_transfers"] == 0 + for peer in results["transfers"]["peers"].values() + ) + # Disconnect after the first streaming token, then verify fresh work. + response = await api.chat.completions.create( + model=uid, **{**bodies()[3], "max_tokens": 1024}, stream=True + ) + async with response: + async for chunk in response: + if chunk.choices and chunk.choices[0].delta.content: + break + await asyncio.sleep(1) + await api.chat.completions.create(model=uid, **bodies()[0]) + await asyncio.sleep(0.1) + results["after_cancel"] = await snapshot(worker, uid, True) + assert results["after_cancel"]["directory"]["active_handoffs"] == 0 + assert all( + peer["active_transfers"] == 0 + for peer in results["after_cancel"]["peers"].values() + ) + # Cancel a prepared room while its long prefill is still running. + request_id = uuid.uuid4().hex + body = bodies()[3] + body["messages"][0]["content"] = ( + "A new uncached prefill. " * 1000 + "Explain evaporation." + ) + body["max_tokens"] = 1024 + body["extra_body"]["request_id"] = request_id + pending = asyncio.create_task( + api.chat.completions.create(model=uid, **body) + ) + directory = await xo.actor_ref(address=worker, uid="xavier-cache-" + uid) + for _ in range(1000): + if (await directory.get_stats())["active_handoffs"]: + break + assert ( + not pending.done() + ), "Request finished before the cancellation probe" + await asyncio.sleep(0.001) + else: + raise AssertionError("Cancellation probe did not reach a prepared room") + results["early_cancel"] = await asyncio.to_thread( + Client(endpoint).abort_request, uid, request_id + ) + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await asyncio.sleep(1) + await api.chat.completions.create(model=uid, **bodies()[0]) + await asyncio.sleep(0.1) + results["after_early_cancel"] = await snapshot(worker, uid, True) + assert results["after_early_cancel"]["directory"]["active_handoffs"] == 0 + assert all( + peer["active_transfers"] == 0 + for peer in results["after_early_cancel"]["peers"].values() + ) + finally: + await api.close() + (root / "results.json").write_text(json.dumps(results, indent=2)) + return results + + +def run(args): + root = args.output_dir.resolve() + root.mkdir(parents=True, exist_ok=True) + isolation = GPUIsolation(args.idle_pid, args.idle_timeout) + isolation.wait_idle() + port = get_next_port() + endpoint = f"http://127.0.0.1:{port}" + env = dict( + os.environ, + XINFERENCE_HOME=str(root / "home"), + XINFERENCE_AUTH_ADVANCED="false", + XINFERENCE_ENABLE_VIRTUAL_ENV="0", + ) + mode = args.mode + engines = ( + mode.split("-") + if mode in ("vllm-sglang", "sglang-vllm") + else [mode.split("-")[-1]] + ) + with (root / "server.log").open("w") as log: + proc = subprocess.Popen( + [ + sys.executable, + "-c", + "from xinference.deploy.cmdline import local; local()", + "--host", + "127.0.0.1", + "--port", + str(port), + "--log-level", + "INFO", + ], + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + try: + for _ in range(120): + if proc.poll() is not None: + raise RuntimeError("Xinference exited; inspect server.log") + try: + client = Client(endpoint) + workers = client.get_workers_info() + if workers: + break + except requests.RequestException: + pass + time.sleep(1) + else: + raise RuntimeError("Server readiness timeout") + worker = workers[0]["work-ip"] + uid = "cross-engine-benchmark" + launch = dict( + model_uid=uid, + model_name="qwen2.5-instruct", + model_size_in_billions="0_5", + model_format="pytorch", + quantization="none", + model_path=str(args.model_path), + model_engine=engines[0], + enable_virtual_env=False, + envs={ + key: env[key] + for key in ("PATH", "PYTHONPATH", "LD_PRELOAD") + if key in env + }, + ) + if mode.startswith("mono-"): + launch.update( + n_gpu=1, gpu_idx=[0], **engine_options(engines[0], "hybrid", args) + ) + else: + if mode.startswith("native-"): + engines = [engines[0]] * 2 + launch["transfer_backend_type"] = "nixl" + launch["replica"] = 2 + launch["replica_config"] = [ + dict( + role=role, + model_engine=engine, + engine_config=engine_options(engine, role, args), + devices=[dict(worker_ip=worker, n_gpu=1, gpu_idx=[index])], + ) + for index, (role, engine) in enumerate( + zip(("prefill", "decode"), engines) + ) + ] + (root / "launch.json").write_text(json.dumps(launch, indent=2)) + (root / "versions.json").write_text( + json.dumps( + { + name: version(name) + for name in ("vllm", "sglang", "torch", "xoscar") + }, + indent=2, + ) + ) + client.launch_model(**launch) + print("launched", mode, flush=True) + with isolation: + result = asyncio.run( + requests_and_measure(endpoint, uid, root, worker, mode, args) + ) + (root / "isolation.json").write_text( + json.dumps( + dict( + samples=isolation.samples, interference=isolation.interference + ), + indent=2, + ) + ) + if isolation.interference: + raise RuntimeError("Foreign GPU interference; discard measurements") + client.terminate_model(uid) + print( + json.dumps( + dict( + mode=mode, + performance=[row["summary"] for row in result["performance"]], + ) + ), + flush=True, + ) + finally: + stop_server(proc) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--mode", + required=True, + choices=[ + "mono-vllm", + "mono-sglang", + "native-vllm", + "native-sglang", + "vllm-sglang", + "sglang-vllm", + ], + ) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--model-path", required=True, type=Path) + parser.add_argument("--kv-tokens", type=int, default=32768) + parser.add_argument("--requests", type=int, default=128) + parser.add_argument("--concurrency", type=int, nargs="+", default=[1, 16]) + parser.add_argument("--idle-pid", type=int, action="append", default=[]) + parser.add_argument("--idle-timeout", type=int, default=7200) + run(parser.parse_args()) diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po index 145403836a..587a25c898 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po @@ -393,10 +393,6 @@ msgstr "SGLang >= ``0.5.21`` akzeptiert dieselbe prefill/decode-``replica_config msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D überträgt KV über Xaviers NIXL-Transport direkt zwischen GPUs. Prefill und decode laufen mit SGLangs nativem P/D-Lebenszyklus gleichzeitig. Quell- und Ziel-GPU-Slots bleiben bis zum Abschluss der Übertragung reserviert; decode erhält auch das erste von prefill gesampelte Token. Streaming- und Nicht-Streaming-Antworten werden unterstützt. Übertragungsfehler führen zu Fehlern." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "Die Modellgrenzen stehen unter :ref:`user_guide_vllm_enhancement`. Installieren Sie ``xoscar[nixl]>=0.11.1`` in Worker- und Modellumgebungen und verwenden Sie erreichbare Worker-Adressen. Ohne NIXL schlägt der Start ohne CPU-Fallback fehl. Verwenden Sie den ``xavier``-Transport; SGLangs eigener Mooncake-Backend und engineübergreifendes P/D werden von dieser Integration nicht angeboten." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D verwendet weder ``xavier_cache_bytes`` noch CPU HiCache. Ein dauerhaft gehaltener Xavier-GPU-Verlauf wird noch nicht unterstützt; ``xavier_gpu_cache_bytes`` kann entfallen oder auf ``0`` gesetzt werden. Starten Sie die Bereitstellung nach einem Worker-Neustart erneut. Vergleichen Sie TTFT und Durchsatz für Ihre Last mit gewöhnlichen Replikaten und nativem SGLang P/D." @@ -504,14 +500,14 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "Verwenden Sie für MLX den Standardtransport ``xavier``; natives NIXL wird " "nicht unterstützt. Alle Worker müssen die Actor-Adresse des Supervisors " "erreichen. Starten Sie das Deployment nach einem Worker-Neustart neu. Zwei " "Replikate auf einem Mac teilen dessen Metal-GPU; messen Sie Ihren Workload, " -"bevor Sie einen Durchsatzvorteil erwarten. Engineübergreifendes P/D " +"bevor Sie einen Durchsatzvorteil erwarten. Engineübergreifendes P/D mit MLX " "einschließlich NVIDIA-zu-Mac-Übergaben wird noch nicht unterstützt." #: user_guide/pd_separation.rst:102 @@ -534,3 +530,82 @@ msgstr "" "gleichzeitigen P/D-Anfragen reservieren. Gewöhnliche Anfragen mit " "gemeinsamem Cache überspringen die Veröffentlichung, wenn ihr Präfix die " "Cache-Kapazität oder die Protokollgrenze von 4096 Seiten überschreitet." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"Die Modellgrenzen stehen in :ref:`user_guide_vllm_enhancement`. Installieren" +" Sie ``xoscar[nixl]>=0.11.1`` in Worker- und Modellumgebungen und verwenden " +"Sie erreichbare Worker-Adressen. Ohne NIXL schlägt der Start ohne CPU-" +"Fallback fehl. Verwenden Sie ``xavier``; SGLangs Mooncake-Backend wird nicht" +" angeboten. Für engineübergreifendes P/D gelten die folgenden Engine-" +"Einstellungen je Replikat." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "Engineübergreifendes vLLM/SGLang P/D" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"Setzen Sie ``model_engine`` und ``engine_config`` in jedem " +"``replica_config``-Eintrag, um je Rolle vLLM oder SGLang auszuwählen. " +"Engineübergreifendes P/D erfordert Xavier-GPU-Übertragung. Engine-Optionen " +"erben die Startargumente; ``engine_config`` überschreibt sie je Replikat." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"Beide Worker benötigen identische lokale Gewichte, Tokenizer-Dateien und " +"Kontextgrenzen. Unterstützt werden unquantisierte FP16-Textmodelle Qwen2, " +"Qwen3 und Llama mit vollständiger Attention und TP=PP=DP=1. Erforderlich " +"sind vLLM >=0.21.0, SGLang >=0.5.21 und ``xoscar[nixl]>=0.11.1``. Xavier " +"prüft Verträge und Prompt-Token-IDs, konvertiert KV-Seiten mit 64 Tokens auf" +" der GPU und meldet Inkompatibilität oder Übertragungsfehler." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"Streaming- und Nicht-Streaming-Anfragen erfordern ``n=1``. SGLang decode " +"verwendet das von prefill gesampelte erste Ausgabetoken. vLLM decode " +"importiert alle KV außer dem letzten Prompt-Token, berechnet dieses Token " +"und sampelt die Ausgabe. Engine-Kernels können kleine numerische " +"Unterschiede erzeugen. vLLM decode benötigt mindestens zwei Prompt-Tokens." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"Engineübergreifendes P/D unterstützt weder CPU-Fallback noch Xavier-Verlauf," +" logprobs, strukturiertes Sampling, LoRA, spekulatives Decoding, multimodale" +" Eingaben oder hybride Attention. Lassen Sie ``xavier_gpu_cache_bytes`` weg " +"oder setzen Sie es auf ``0``; setzen Sie ``xavier_cache_bytes`` nicht. Nach " +"einem Worker-Neustart ist eine erneute Bereitstellung nötig. Native NIXL-" +"Vergleiche verwenden dieselbe Engine für beide Rollen. MLX wird nicht " +"unterstützt." diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po index 3068bce31c..0133c7c09f 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po @@ -389,10 +389,6 @@ msgstr "SGLang >= ``0.5.21`` acepta el mismo ``replica_config`` prefill/decode m msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D transfiere KV directamente entre GPU mediante el transporte NIXL de Xavier. Prefill y decode se ejecutan simultáneamente con el ciclo P/D nativo de SGLang. Los slots GPU de origen y destino mantienen su reserva hasta completar la transferencia; decode también recibe el primer token muestreado por prefill. Se admiten respuestas con y sin streaming. Los fallos de transferencia generan errores." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "Las limitaciones del modelo se describen en :ref:`user_guide_vllm_enhancement`. Instale ``xoscar[nixl]>=0.11.1`` en los entornos worker y modelo y use direcciones worker accesibles. Sin NIXL, el lanzamiento falla sin alternativa CPU. Use el transporte ``xavier``; esta integración no expone el backend Mooncake propio de SGLang ni el P/D entre motores." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D no usa ``xavier_cache_bytes`` ni CPU HiCache. Aún no se admite conservar un historial GPU de Xavier; omita ``xavier_gpu_cache_bytes`` o establézcalo en ``0``. Reinicie el despliegue tras reiniciar un worker. Compare TTFT y rendimiento con réplicas normales y P/D nativo de SGLang para su carga." @@ -499,15 +495,15 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "Use el transporte predeterminado ``xavier`` para MLX; NIXL nativo no es " "compatible. Todos los workers deben acceder a la dirección actor del " "supervisor. Reinicie el despliegue tras reiniciar un worker. Dos réplicas en" " un Mac comparten su GPU Metal; mida su carga antes de esperar mejoras de " -"rendimiento. El P/D entre motores, incluida la transferencia de NVIDIA a " -"Mac, aún no es compatible." +"rendimiento. El P/D entre motores con MLX, incluida la transferencia de " +"NVIDIA a Mac, aún no es compatible." #: user_guide/pd_separation.rst:102 msgid "" @@ -529,3 +525,80 @@ msgstr "" "Las solicitudes ordinarias de caché compartida omiten la publicación cuando " "su prefijo supera la capacidad de caché o el límite de protocolo de 4096 " "páginas." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"Las limitaciones se describen en :ref:`user_guide_vllm_enhancement`. Instale" +" ``xoscar[nixl]>=0.11.1`` en los entornos del worker y del modelo y use " +"direcciones accesibles. Sin NIXL, el lanzamiento falla sin retorno a CPU. " +"Use ``xavier``; el backend Mooncake de SGLang no se expone. Para P/D entre " +"motores, use las opciones por réplica siguientes." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "P/D vLLM/SGLang entre motores" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"Configure ``model_engine`` y ``engine_config`` en cada entrada " +"``replica_config`` para elegir vLLM o SGLang por rol. El P/D entre motores " +"requiere transferencia GPU Xavier. Las opciones heredan los argumentos de " +"lanzamiento; ``engine_config`` los sobrescribe por réplica." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"Ambos workers requieren pesos locales, archivos de tokenizer y límites de " +"contexto idénticos. Se admiten modelos de texto Qwen2, Qwen3 y Llama FP16 " +"sin cuantizar y con atención completa, TP=PP=DP=1. Requiere vLLM >=0.21.0, " +"SGLang >=0.5.21 y ``xoscar[nixl]>=0.11.1``. Xavier comprueba contratos e ID " +"de tokens del prompt, convierte páginas KV de 64 tokens en GPU y genera " +"errores ante incompatibilidades o fallos de transferencia." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"Las solicitudes con y sin streaming requieren ``n=1``. SGLang decode utiliza" +" el primer token de salida muestreado por prefill. vLLM decode importa todos" +" los KV excepto el último token del prompt, calcula ese token y muestrea la " +"salida. Los kernels pueden producir pequeñas diferencias numéricas. vLLM " +"decode requiere al menos dos tokens de prompt." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"El P/D entre motores no admite retorno a CPU, historial Xavier, logprobs, " +"muestreo estructurado, LoRA, decodificación especulativa, entradas " +"multimodales ni atención híbrida. Omita ``xavier_gpu_cache_bytes`` o " +"establézcalo en ``0``; no configure ``xavier_cache_bytes``. Vuelva a " +"desplegar después de reiniciar un worker. Las comparaciones NIXL nativas " +"usan el mismo motor en ambos roles. MLX no es compatible." diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po index a26fa13e26..c2b6406c7f 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po @@ -393,10 +393,6 @@ msgstr "SGLang >= ``0.5.21`` accepte le même ``replica_config`` prefill/decode msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D transfère directement le KV entre GPU via le transport NIXL de Xavier. Prefill et decode s'exécutent simultanément avec le cycle P/D natif de SGLang. Les emplacements GPU source et destination restent réservés jusqu'à la fin du transfert ; decode reçoit aussi le premier token échantillonné par prefill. Les réponses avec et sans streaming sont prises en charge. Les échecs de transfert provoquent des erreurs." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "Les limites des modèles sont décrites dans :ref:`user_guide_vllm_enhancement`. Installez ``xoscar[nixl]>=0.11.1`` dans les environnements worker et modèle, avec des adresses worker accessibles. L'absence de NIXL empêche le lancement sans repli CPU. Utilisez le transport ``xavier`` ; cette intégration n'expose pas le backend Mooncake propre à SGLang ni le P/D entre moteurs." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D n'utilise ni ``xavier_cache_bytes`` ni CPU HiCache. La conservation d'un historique GPU Xavier n'est pas encore prise en charge ; omettez ``xavier_gpu_cache_bytes`` ou définissez-le sur ``0``. Relancez le déploiement après le redémarrage d'un worker. Comparez le TTFT et le débit avec des répliques ordinaires et le P/D SGLang natif pour votre charge." @@ -506,15 +502,15 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "Utilisez le transport par défaut ``xavier`` pour MLX ; NIXL natif n’est pas " "pris en charge. Tous les workers doivent pouvoir joindre l’adresse actor du " "supervisor. Relancez le déploiement après le redémarrage d’un worker. Deux " "répliques sur un même Mac partagent son GPU Metal ; mesurez votre charge " -"avant d’attendre un gain de débit. Le P/D entre moteurs, notamment de NVIDIA" -" vers Mac, n’est pas encore pris en charge." +"avant d’attendre un gain de débit. Le P/D entre moteurs impliquant MLX, " +"notamment de NVIDIA vers Mac, n’est pas encore pris en charge." #: user_guide/pd_separation.rst:102 msgid "" @@ -536,3 +532,82 @@ msgstr "" "requêtes P/D simultanées. Les requêtes ordinaires avec cache partagé " "omettent la publication lorsque leur préfixe dépasse la capacité du cache ou" " la limite du protocole de 4096 pages." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"Les limitations sont décrites dans :ref:`user_guide_vllm_enhancement`. " +"Installez ``xoscar[nixl]>=0.11.1`` dans les environnements worker et modèle " +"et utilisez des adresses accessibles. Sans NIXL, le lancement échoue sans " +"repli CPU. Utilisez ``xavier`` ; le backend Mooncake de SGLang n’est pas " +"exposé. Pour le P/D entre moteurs, utilisez les paramètres par réplique ci-" +"dessous." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "P/D vLLM/SGLang entre moteurs" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"Définissez ``model_engine`` et ``engine_config`` dans chaque entrée " +"``replica_config`` pour choisir vLLM ou SGLang par rôle. Le P/D entre " +"moteurs exige le transfert GPU Xavier. Les options héritent des arguments de" +" lancement ; ``engine_config`` les remplace par réplique." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"Les deux workers exigent les mêmes poids locaux, fichiers de tokenizer et " +"limites de contexte. Les modèles pris en charge sont Qwen2, Qwen3 et Llama " +"textuels FP16 non quantifiés à attention complète avec TP=PP=DP=1. vLLM " +">=0.21.0, SGLang >=0.5.21 et ``xoscar[nixl]>=0.11.1`` sont requis. Xavier " +"vérifie les contrats et les ID des tokens du prompt, convertit les pages KV " +"de 64 tokens sur GPU et signale les incompatibilités ou échecs de transfert." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"Les requêtes avec ou sans streaming exigent ``n=1``. SGLang decode utilise " +"le premier token de sortie échantillonné par prefill. vLLM decode importe " +"tous les KV sauf le dernier token du prompt, calcule ce token et " +"échantillonne la sortie. Les kernels peuvent produire de faibles différences" +" numériques. vLLM decode exige au moins deux tokens de prompt." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"Le P/D entre moteurs ne prend pas en charge le repli CPU, l’historique " +"Xavier, logprobs, l’échantillonnage structuré, LoRA, le décodage spéculatif," +" les entrées multimodales ni l’attention hybride. Omettez " +"``xavier_gpu_cache_bytes`` ou réglez-le à ``0`` ; ne définissez pas " +"``xavier_cache_bytes``. Redéployez après un redémarrage du worker. Les " +"comparaisons NIXL natives utilisent le même moteur pour les deux rôles. MLX " +"n’est pas pris en charge." diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po index a34d329394..f1c2cbffec 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po @@ -391,10 +391,6 @@ msgstr "SGLang >= ``0.5.21`` accetta lo stesso ``replica_config`` prefill/decode msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D trasferisce KV direttamente tra GPU tramite il trasporto NIXL di Xavier. Prefill e decode vengono eseguiti contemporaneamente con il ciclo P/D nativo di SGLang. Gli slot GPU di origine e destinazione rimangono riservati fino al termine del trasferimento; decode riceve anche il primo token campionato da prefill. Sono supportate risposte con e senza streaming. I trasferimenti falliti generano errori." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "Le limitazioni dei modelli sono descritte in :ref:`user_guide_vllm_enhancement`. Installare ``xoscar[nixl]>=0.11.1`` negli ambienti worker e modello e usare indirizzi worker raggiungibili. L'assenza di NIXL impedisce l'avvio senza ripiego sulla CPU. Usare il trasporto ``xavier``; questa integrazione non espone il backend Mooncake proprio di SGLang né il P/D tra motori." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D non usa ``xavier_cache_bytes`` né CPU HiCache. La conservazione della cronologia GPU Xavier non è ancora supportata; omettere ``xavier_gpu_cache_bytes`` o impostarlo su ``0``. Riavviare il deployment dopo il riavvio di un worker. Confrontare TTFT e throughput con repliche ordinarie e P/D nativo di SGLang per il proprio carico." @@ -501,15 +497,15 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "Usare il trasporto predefinito ``xavier`` per MLX; NIXL nativo non è " "supportato. Tutti i worker devono raggiungere l’indirizzo actor del " "supervisor. Riavviare il deployment dopo il riavvio di un worker. Due " "repliche sullo stesso Mac condividono la GPU Metal; misurare il carico prima" -" di aspettarsi un aumento del throughput. Il P/D tra motori, incluso il " -"trasferimento da NVIDIA a Mac, non è ancora supportato." +" di aspettarsi un aumento del throughput. Il P/D tra motori che coinvolge " +"MLX, incluso il trasferimento da NVIDIA a Mac, non è ancora supportato." #: user_guide/pd_separation.rst:102 msgid "" @@ -531,3 +527,80 @@ msgstr "" "richieste P/D simultanee. Le richieste ordinarie con cache condivisa saltano" " la pubblicazione quando il prefisso supera la capacità della cache o il " "limite del protocollo di 4096 pagine." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"I limiti sono descritti in :ref:`user_guide_vllm_enhancement`. Installare " +"``xoscar[nixl]>=0.11.1`` negli ambienti worker e modello e usare indirizzi " +"raggiungibili. Senza NIXL, l’avvio fallisce senza fallback CPU. Usare " +"``xavier``; il backend Mooncake di SGLang non è esposto. Per il P/D tra " +"motori, usare le impostazioni per replica riportate sotto." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "P/D vLLM/SGLang tra motori" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"Impostare ``model_engine`` e ``engine_config`` in ogni voce " +"``replica_config`` per scegliere vLLM o SGLang per ciascun ruolo. Il P/D tra" +" motori richiede il trasferimento GPU Xavier. Le opzioni ereditano gli " +"argomenti di avvio; ``engine_config`` li sovrascrive per replica." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"Entrambi i worker richiedono pesi locali, file del tokenizer e limiti di " +"contesto identici. Sono supportati modelli di testo Qwen2, Qwen3 e Llama " +"FP16 non quantizzati ad attenzione completa con TP=PP=DP=1. Sono richiesti " +"vLLM >=0.21.0, SGLang >=0.5.21 e ``xoscar[nixl]>=0.11.1``. Xavier verifica " +"contratti e ID dei token del prompt, converte pagine KV da 64 token sulla " +"GPU e segnala incompatibilità o errori di trasferimento." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"Le richieste con e senza streaming richiedono ``n=1``. SGLang decode usa il " +"primo token di output campionato da prefill. vLLM decode importa tutti i KV " +"tranne l’ultimo token del prompt, calcola quel token e campiona l’output. I " +"kernel possono produrre piccole differenze numeriche. vLLM decode richiede " +"almeno due token di prompt." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"Il P/D tra motori non supporta fallback CPU, cronologia Xavier, logprobs, " +"campionamento strutturato, LoRA, decodifica speculativa, input multimodali o" +" attenzione ibrida. Omettere ``xavier_gpu_cache_bytes`` o impostarlo a " +"``0``; non impostare ``xavier_cache_bytes``. Dopo il riavvio di un worker è " +"necessario ridistribuire. I confronti NIXL nativi usano lo stesso motore per" +" entrambi i ruoli. MLX non è supportato." diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po index dfa74ec4bd..a3233ae909 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po @@ -353,10 +353,6 @@ msgstr "SGLang >= ``0.5.21`` では、Python クライアント、REST API、CLI msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D は Xavier の NIXL トランスポートで GPU 間に KV を直接転送します。Prefill と decode は SGLang ネイティブの P/D ライフサイクルで並行実行されます。転送完了まで送信元と送信先の GPU スロットの所有権を保持し、decode は prefill がサンプリングした最初のトークンも受け取ります。ストリーミングと非ストリーミングに対応し、転送失敗はエラーになります。" -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "モデルの制限は :ref:`user_guide_vllm_enhancement` を参照してください。worker とモデル環境の両方に ``xoscar[nixl]>=0.11.1`` をインストールし、接続可能な worker アドレスを使用してください。NIXL がない場合は起動に失敗し、CPU へフォールバックしません。``xavier`` トランスポートを使用してください。この統合では SGLang 独自の Mooncake バックエンドとエンジン間 P/D は公開していません。" - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D は ``xavier_cache_bytes`` や CPU HiCache を使用しません。Xavier GPU 履歴キャッシュの保持は未対応で、``xavier_gpu_cache_bytes`` は省略するか ``0`` に設定してください。worker の再起動後はデプロイを再起動してください。実際の負荷で通常のレプリカとネイティブ SGLang P/D に対する TTFT とスループットを測定してください。" @@ -452,13 +448,13 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "MLX では既定の ``xavier`` 転送を使用します。ネイティブ NIXL は未対応です。全 worker から supervisor の " "actor アドレスに接続できる必要があります。Worker 再起動後はデプロイを再起動してください。同じ Mac 上の 2 レプリカは Metal " -"GPU を共有するため、スループット改善は実際の負荷で測定してください。NVIDIA から Mac への引き継ぎを含む異なるエンジン間の P/D " -"は未対応です。" +"GPU を共有するため、スループット改善は実際の負荷で測定してください。NVIDIA から Mac への引き継ぎを含む MLX " +"に関わる異なるエンジン間の P/D は未対応です。" #: user_guide/pd_separation.rst:102 msgid "" @@ -477,3 +473,73 @@ msgstr "" " リクエスト全体が予約する異なるページ数に合わせて ``xavier_cache_bytes`` " "を設定してください。通常の共有キャッシュリクエストは、プレフィックスがキャッシュ容量またはプロトコルの 4096 " "ページ制限を超える場合、公開を省略します。" + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"モデルの制限は :ref:`user_guide_vllm_enhancement` を参照してください。ワーカーとモデル環境に " +"``xoscar[nixl]>=0.11.1`` をインストールし、到達可能なアドレスを使います。NIXL がなければ CPU " +"にフォールバックせず起動が失敗します。``xavier`` を使います。SGLang の Mooncake バックエンドは公開されていません。エンジン間" +" P/D は以下のレプリカごとの設定を使います。" + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "エンジン間の vLLM/SGLang P/D" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"各 ``replica_config`` に ``model_engine`` と ``engine_config`` を設定し、役割ごとに vLLM " +"または SGLang を選択します。エンジン間 P/D には Xavier GPU " +"転送が必要です。エンジンの設定は起動引数を継承し、``engine_config`` がレプリカごとに上書きします。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"両ワーカーで同一のローカル重み、tokenizer ファイル、コンテキスト上限を使用します。TP=PP=DP=1 の非量子化 FP16 全注意 " +"Qwen2、Qwen3、Llama テキストモデルをサポートします。vLLM >=0.21.0、SGLang " +">=0.5.21、``xoscar[nixl]>=0.11.1`` が必要です。Xavier は契約と実際の prompt token ID " +"を検証し、GPU 上で 64-token KV ページを変換します。非互換や転送失敗はエラーになります。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"ストリーミングと非ストリーミングの要求は ``n=1`` が必要です。SGLang decode は prefill がサンプリングした最初の出力 " +"token を使います。vLLM decode は最後の prompt token を除く KV を取り込み、その token " +"を計算して出力をサンプリングします。エンジンのカーネルによって小さな数値差が生じる場合があります。vLLM decode には最低 2 個の " +"prompt token が必要です。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"エンジン間 P/D は CPU フォールバック、Xavier " +"履歴、logprobs、構造化サンプリング、LoRA、投機的デコード、マルチモーダル入力、ハイブリッド注意をサポートしません。``xavier_gpu_cache_bytes``" +" は省略するか ``0`` にし、``xavier_cache_bytes`` は設定しないでください。ワーカー再起動後は再デプロイします。ネイティブ " +"NIXL の比較では両役割に同一エンジンを使います。MLX は未対応です。" diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po index c3a1c9ca08..5ff1039a3e 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po @@ -352,10 +352,6 @@ msgstr "SGLang >= ``0.5.21`` 은 Python 클라이언트, REST API, CLI 및 Web U msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D는 Xavier의 NIXL 전송으로 GPU 간에 KV를 직접 전달합니다. Prefill과 decode는 SGLang 네이티브 P/D 수명 주기로 동시에 실행됩니다. 전송이 완료될 때까지 소스와 대상 GPU 슬롯의 소유권을 유지하며, decode는 prefill이 샘플링한 첫 토큰도 받습니다. 스트리밍 및 비스트리밍 응답을 지원하며 전송 실패는 오류가 됩니다." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "모델 제한은 :ref:`user_guide_vllm_enhancement` 를 참조하세요. worker와 모델 환경에 ``xoscar[nixl]>=0.11.1`` 을 설치하고 접근 가능한 worker 주소를 사용하세요. NIXL이 없으면 CPU로 대체하지 않고 시작에 실패합니다. ``xavier`` 전송을 사용하세요. 이 통합에서는 SGLang 자체 Mooncake 백엔드와 엔진 간 P/D를 제공하지 않습니다." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D는 ``xavier_cache_bytes`` 나 CPU HiCache를 사용하지 않습니다. Xavier GPU 이력 캐시 보존은 아직 지원하지 않으며 ``xavier_gpu_cache_bytes`` 는 생략하거나 ``0`` 으로 설정하세요. worker 재시작 후 배포를 다시 시작하세요. 실제 워크로드에서 일반 복제본 및 네이티브 SGLang P/D와 TTFT 및 처리량을 비교하세요." @@ -451,13 +447,13 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "MLX에는 기본 ``xavier`` 전송을 사용하세요. 네이티브 NIXL은 지원하지 않습니다. 모든 worker는 supervisor의 " "actor 주소에 접근할 수 있어야 합니다. Worker 재시작 후 배포를 다시 시작하세요. 같은 Mac의 두 복제본은 Metal " -"GPU를 공유하므로 처리량 향상을 기대하기 전에 워크로드를 측정하세요. NVIDIA에서 Mac으로의 인계를 포함한 엔진 간 P/D는 아직" -" 지원하지 않습니다." +"GPU를 공유하므로 처리량 향상을 기대하기 전에 워크로드를 측정하세요. NVIDIA에서 Mac으로의 인계를 포함한 MLX 관련 엔진 간 " +"P/D는 아직 지원하지 않습니다." #: user_guide/pd_separation.rst:102 msgid "" @@ -475,3 +471,73 @@ msgstr "" " 512 MiB는 56페이지, 즉 접두사 3584토큰을 저장합니다. 각 접두사를 전체 페이지 단위로 올림하고 모든 동시 P/D 요청이 " "예약하는 서로 다른 페이지에 맞게 ``xavier_cache_bytes`` 를 설정하십시오. 일반 공유 캐시 요청은 접두사가 캐시 용량 " "또는 프로토콜의 4096페이지 한도를 초과하면 게시를 건너뜁니다." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"모델 제한은 :ref:`user_guide_vllm_enhancement` 를 참조하세요. 워커와 모델 환경에 " +"``xoscar[nixl]>=0.11.1`` 을 설치하고 접근 가능한 주소를 사용하세요. NIXL이 없으면 CPU 폴백 없이 실행이 " +"실패합니다. ``xavier`` 를 사용하세요. SGLang의 Mooncake 백엔드는 제공되지 않습니다. 엔진 간 P/D는 아래의 " +"복제본별 설정을 사용하세요." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "엔진 간 vLLM/SGLang P/D" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"각 ``replica_config`` 항목에 ``model_engine`` 과 ``engine_config`` 를 설정하여 역할별로 " +"vLLM 또는 SGLang을 선택합니다. 엔진 간 P/D에는 Xavier GPU 전송이 필요합니다. 엔진 옵션은 실행 인수를 상속하며 " +"``engine_config`` 가 복제본별로 재정의합니다." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"두 워커는 동일한 로컬 가중치, tokenizer 파일 및 컨텍스트 한도를 사용해야 합니다. TP=PP=DP=1인 비양자화 FP16 전체" +" 어텐션 Qwen2, Qwen3 및 Llama 텍스트 모델을 지원합니다. vLLM >=0.21.0, SGLang >=0.5.21 및 " +"``xoscar[nixl]>=0.11.1`` 이 필요합니다. Xavier는 계약과 실제 prompt token ID를 확인하고 GPU에서" +" 64-token KV 페이지를 변환합니다. 비호환 또는 전송 실패 시 오류가 발생합니다." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"스트리밍 및 비스트리밍 요청은 ``n=1`` 이 필요합니다. SGLang decode는 prefill이 샘플링한 첫 출력 token을 " +"사용합니다. vLLM decode는 마지막 prompt token을 제외한 KV를 가져오고 해당 token을 계산한 다음 출력을 " +"샘플링합니다. 엔진 커널에 따라 작은 수치 차이가 발생할 수 있습니다. vLLM decode에는 최소 두 개의 prompt token이 " +"필요합니다." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"엔진 간 P/D는 CPU 폴백, Xavier 기록 캐시, logprobs, 구조화 샘플링, LoRA, 추측 디코딩, 멀티모달 입력 또는 " +"하이브리드 어텐션을 지원하지 않습니다. ``xavier_gpu_cache_bytes`` 를 생략하거나 ``0`` 으로 설정하고 " +"``xavier_cache_bytes`` 는 설정하지 마세요. 워커 재시작 후 다시 배포하세요. 네이티브 NIXL 비교는 두 역할에 같은" +" 엔진을 사용합니다. MLX는 지원하지 않습니다." diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po index f44947531c..94622a9e6e 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po @@ -387,10 +387,6 @@ msgstr "SGLang >= ``0.5.21`` aceita o mesmo ``replica_config`` prefill/decode pe msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D transfere KV diretamente entre GPUs pelo transporte NIXL do Xavier. Prefill e decode executam simultaneamente usando o ciclo P/D nativo do SGLang. Os slots GPU de origem e destino permanecem reservados até o fim da transferência; decode também recebe o primeiro token amostrado pelo prefill. Respostas com e sem streaming são suportadas. Falhas de transferência geram erros." -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "As limitações dos modelos estão em :ref:`user_guide_vllm_enhancement`. Instale ``xoscar[nixl]>=0.11.1`` nos ambientes worker e modelo e use endereços worker acessíveis. Sem NIXL, o lançamento falha sem alternativa CPU. Use o transporte ``xavier``; esta integração não expõe o backend Mooncake próprio do SGLang nem P/D entre mecanismos." - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D não usa ``xavier_cache_bytes`` nem CPU HiCache. A retenção do histórico GPU Xavier ainda não é suportada; omita ``xavier_gpu_cache_bytes`` ou defina como ``0``. Reinicie a implantação após reiniciar um worker. Compare TTFT e taxa de transferência com réplicas comuns e P/D nativo do SGLang para sua carga." @@ -496,14 +492,15 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "Use o transporte padrão ``xavier`` para MLX; NIXL nativo não é suportado. " "Todos os workers devem alcançar o endereço actor do supervisor. Reinicie o " "deployment após reiniciar um worker. Duas réplicas no mesmo Mac compartilham" " a GPU Metal; meça sua carga antes de esperar ganho de throughput. P/D entre" -" engines, incluindo transferência de NVIDIA para Mac, ainda não é suportado." +" engines envolvendo MLX, incluindo transferência de NVIDIA para Mac, ainda " +"não é suportado." #: user_guide/pd_separation.rst:102 msgid "" @@ -524,3 +521,80 @@ msgstr "" "distintas reservadas por todas as solicitações P/D simultâneas. Solicitações" " comuns de cache compartilhado pulam a publicação quando o prefixo excede a " "capacidade do cache ou o limite de protocolo de 4096 páginas." + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"As limitações estão em :ref:`user_guide_vllm_enhancement`. Instale " +"``xoscar[nixl]>=0.11.1`` nos ambientes do worker e do modelo e use endereços" +" acessíveis. Sem NIXL, a inicialização falha sem fallback para CPU. Use " +"``xavier``; o backend Mooncake do SGLang não é exposto. Para P/D entre " +"mecanismos, use as opções por réplica abaixo." + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "P/D vLLM/SGLang entre mecanismos" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"Defina ``model_engine`` e ``engine_config`` em cada entrada " +"``replica_config`` para escolher vLLM ou SGLang por função. P/D entre " +"mecanismos exige transferência GPU Xavier. As opções herdam os argumentos de" +" inicialização; ``engine_config`` os substitui por réplica." + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"Os dois workers exigem pesos locais, arquivos do tokenizer e limites de " +"contexto idênticos. São compatíveis os modelos de texto Qwen2, Qwen3 e Llama" +" FP16 não quantizados com atenção completa e TP=PP=DP=1. Exige vLLM " +">=0.21.0, SGLang >=0.5.21 e ``xoscar[nixl]>=0.11.1``. Xavier verifica " +"contratos e IDs dos tokens do prompt, converte páginas KV de 64 tokens na " +"GPU e gera erros em incompatibilidades ou falhas de transferência." + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"Solicitações com e sem streaming exigem ``n=1``. SGLang decode usa o " +"primeiro token de saída amostrado por prefill. vLLM decode importa todos os " +"KV exceto o último token do prompt, calcula esse token e amostra a saída. Os" +" kernels podem produzir pequenas diferenças numéricas. vLLM decode exige " +"pelo menos dois tokens de prompt." + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"P/D entre mecanismos não oferece fallback para CPU, histórico Xavier, " +"logprobs, amostragem estruturada, LoRA, decodificação especulativa, entradas" +" multimodais ou atenção híbrida. Omita ``xavier_gpu_cache_bytes`` ou " +"defina-o como ``0``; não defina ``xavier_cache_bytes``. Reimplante após " +"reiniciar um worker. Comparações NIXL nativas usam o mesmo mecanismo nas " +"duas funções. MLX não é compatível." diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po index 8ce58e505a..c2cae5aac6 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po @@ -346,10 +346,6 @@ msgstr "SGLang >= ``0.5.21`` 可通过 Python 客户端、REST API、CLI 和 Web msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D 通过 Xavier 的 NIXL 传输在 GPU 之间直接传递 KV。Prefill 和 decode 使用 SGLang 原生 P/D 生命周期并行运行。源和目标 GPU 槽位保留所有权，直到传输完成；decode 还会接收 prefill 采样的第一个 token。支持流式和非流式响应。传输失败会报错。" -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "模型限制见 :ref:`user_guide_vllm_enhancement`。在 worker 和模型环境中安装 ``xoscar[nixl]>=0.11.1``，并使用可访问的 worker 地址。缺少 NIXL 时启动失败，不会回退到 CPU。使用 ``xavier`` 传输；此集成尚未暴露 SGLang 自身的 Mooncake 后端和跨引擎 P/D。" - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D 不使用 ``xavier_cache_bytes`` 或 CPU HiCache。尚不支持保留 Xavier GPU 历史缓存；可以省略 ``xavier_gpu_cache_bytes`` 或将其设为 ``0``。Worker 重启后需重新启动部署。请针对实际负载，对比普通副本和原生 SGLang P/D 的 TTFT 与吞吐量。" @@ -442,12 +438,12 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "MLX 使用默认的 ``xavier`` 传输，不支持原生 NIXL。所有 worker 必须能访问 supervisor 的 actor " -"地址。Worker 重启后需要重新启动部署。同一台 Mac 上的两个副本共享 Metal GPU，应先测量工作负载，再评估吞吐收益。目前不支持跨引擎 " -"P/D，包括 NVIDIA 到 Mac 的交接。" +"地址。Worker 重启后需要重新启动部署。同一台 Mac 上的两个副本共享 Metal GPU，应先测量工作负载，再评估吞吐收益。目前不支持涉及 " +"MLX 的跨引擎 P/D，包括 NVIDIA 到 Mac 的交接。" #: user_guide/pd_separation.rst:102 msgid "" @@ -464,3 +460,70 @@ msgstr "" " 字节（K 和 V，每个值占两字节）。例如，36 层、8 个 KV 头、头维度为 128 时，每页需要 9 MiB；512 MiB 可保存 56 页，即" " 3584 个前缀 token。将每个前缀向上取整到整页，并按所有并发 P/D 请求预留的不同页总量配置 " "``xavier_cache_bytes``。普通共享缓存请求的前缀超过缓存容量或 4096 页协议上限时，会跳过发布。" + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"模型限制参见 :ref:`user_guide_vllm_enhancement`。在 worker 和模型环境中安装 " +"``xoscar[nixl]>=0.11.1``，使用可访问的 worker 地址。缺少 NIXL 时启动失败，不回退到 CPU。使用 " +"``xavier`` 传输；不开放 SGLang 自带的 Mooncake 后端。跨引擎 P/D 使用下方的逐副本引擎配置。" + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "跨引擎 vLLM/SGLang P/D" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"在每个 ``replica_config`` 条目中设置 ``model_engine`` 和 ``engine_config``，分别为各角色选择 " +"vLLM 或 SGLang。跨引擎 P/D 需要 Xavier GPU 传输。引擎选项继承启动参数，``engine_config`` " +"为各副本覆盖这些选项。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"两个 worker 必须使用相同的本地权重、tokenizer 文件及上下文限制。支持 TP=PP=DP=1 的未量化 FP16 全注意力 " +"Qwen2、Qwen3 和 Llama 文本模型。需要 vLLM >=0.21.0、SGLang >=0.5.21 及 " +"``xoscar[nixl]>=0.11.1``。Xavier 校验契约与实际 prompt token ID，在 GPU 上转换 64-token " +"KV 页；不兼容或传输失败会报错。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"流式与非流式请求均要求 ``n=1``。SGLang decode 使用 prefill 采样的首个输出 token。vLLM decode " +"导入最后一个 prompt token 之前的 KV，计算该 token 后采样输出。引擎算子可能产生小幅数值差异。vLLM decode 要求至少两个" +" prompt token。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"跨引擎 P/D 不支持 CPU 回退、Xavier 历史缓存、logprobs、结构化采样、LoRA、推测解码、多模态输入或混合注意力。省略 " +"``xavier_gpu_cache_bytes`` 或设为 ``0``，不要设置 ``xavier_cache_bytes``。worker " +"重启后需要重新部署。原生 NIXL 对照在两个角色上使用相同引擎。不支持 MLX。" diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po index c5bc3f835a..86f2012197 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po @@ -344,10 +344,6 @@ msgstr "SGLang >= ``0.5.21`` 可透過 Python 用戶端、REST API、CLI 和 Web msgid "SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors." msgstr "SGLang Xavier P/D 透過 Xavier 的 NIXL 傳輸在 GPU 之間直接傳遞 KV。Prefill 和 decode 使用 SGLang 原生 P/D 生命週期並行執行。來源和目的 GPU 槽位保留所有權，直到傳輸完成；decode 也會接收 prefill 取樣的第一個 token。支援串流和非串流回應。傳輸失敗會報錯。" -#: source/user_guide/pd_separation.rst -msgid "The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration." -msgstr "模型限制見 :ref:`user_guide_vllm_enhancement`。在 worker 和模型環境中安裝 ``xoscar[nixl]>=0.11.1``，並使用可連線的 worker 位址。缺少 NIXL 時啟動失敗，不會回退到 CPU。使用 ``xavier`` 傳輸；此整合尚未提供 SGLang 自身的 Mooncake 後端與跨引擎 P/D。" - #: source/user_guide/pd_separation.rst msgid "SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload." msgstr "SGLang GPU P/D 不使用 ``xavier_cache_bytes`` 或 CPU HiCache。尚不支援保留 Xavier GPU 歷史快取；可以省略 ``xavier_gpu_cache_bytes`` 或將其設為 ``0``。Worker 重新啟動後需重新啟動部署。請針對實際負載，比較一般副本與原生 SGLang P/D 的 TTFT 和吞吐量。" @@ -440,12 +436,12 @@ msgid "" "Use the default ``xavier`` transport for MLX; native NIXL is unsupported. " "All workers must reach the supervisor's actor address. Relaunch after a " "worker restart. Two replicas on one Mac share its Metal GPU; measure your " -"workload before expecting a throughput benefit. Cross-engine P/D, including " -"NVIDIA-to-Mac handoff, is not yet supported." +"workload before expecting a throughput benefit. Cross-engine P/D involving " +"MLX, including NVIDIA-to-Mac handoff, is not yet supported." msgstr "" "MLX 使用預設的 ``xavier`` 傳輸，不支援原生 NIXL。所有 worker 必須能存取 supervisor 的 actor " -"位址。Worker 重新啟動後需要重新啟動部署。同一台 Mac 上的兩個副本共用 Metal GPU，應先測量工作負載，再評估吞吐收益。目前不支援跨引擎" -" P/D，包括 NVIDIA 到 Mac 的交接。" +"位址。Worker 重新啟動後需要重新啟動部署。同一台 Mac 上的兩個副本共用 Metal GPU，應先測量工作負載，再評估吞吐收益。目前不支援涉及 " +"MLX 的跨引擎 P/D，包括 NVIDIA 到 Mac 的交接。" #: user_guide/pd_separation.rst:102 msgid "" @@ -462,3 +458,70 @@ msgstr "" " 位元組（K 與 V，每個值占兩個位元組）。例如，36 層、8 個 KV 頭、頭維度為 128 時，每頁需要 9 MiB；512 MiB 可儲存 56 " "頁，即 3584 個前綴 token。將每個前綴向上取整到整頁，並按所有並行 P/D 請求保留的不同頁總量設定 " "``xavier_cache_bytes``。一般共享快取請求的前綴超過快取容量或 4096 頁協定上限時，會跳過發佈。" + + +#: source/user_guide/pd_separation.rst +msgid "" +"The model limitations are described in :ref:`user_guide_vllm_enhancement`. " +"Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use " +"reachable worker addresses. Missing NIXL fails launch without CPU fallback. " +"Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed " +"by this integration. For cross-engine P/D, use the per-replica engine " +"settings below." +msgstr "" +"模型限制請參閱 :ref:`user_guide_vllm_enhancement`。在 worker 和模型環境中安裝 " +"``xoscar[nixl]>=0.11.1``，使用可存取的 worker 位址。缺少 NIXL 時啟動失敗，不回退至 CPU。使用 " +"``xavier`` 傳輸；不開放 SGLang 自帶的 Mooncake 後端。跨引擎 P/D 使用下方的逐副本引擎設定。" + +#: source/user_guide/pd_separation.rst +msgid "Cross-engine vLLM/SGLang P/D" +msgstr "跨引擎 vLLM/SGLang P/D" + +#: source/user_guide/pd_separation.rst +msgid "" +"Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry " +"to select vLLM or SGLang independently for each role. Cross-engine P/D " +"requires Xavier GPU transport. Engine options inherit launch arguments; " +"``engine_config`` overrides them per replica." +msgstr "" +"在每個 ``replica_config`` 項目中設定 ``model_engine`` 和 ``engine_config``，分別為各角色選擇 " +"vLLM 或 SGLang。跨引擎 P/D 需要 Xavier GPU 傳輸。引擎選項繼承啟動參數，``engine_config`` " +"為各副本覆寫這些選項。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Both workers require identical local weights, tokenizer assets and context " +"limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 " +"and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang " +">=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt " +"token IDs, converts 64-token KV pages on GPU and raises errors on " +"incompatibility or transfer failure." +msgstr "" +"兩個 worker 必須使用相同的本機權重、tokenizer 檔案及上下文限制。支援 TP=PP=DP=1 的未量化 FP16 全注意力 " +"Qwen2、Qwen3 和 Llama 文字模型。需要 vLLM >=0.21.0、SGLang >=0.5.21 及 " +"``xoscar[nixl]>=0.11.1``。Xavier 檢查契約與實際 prompt token ID，在 GPU 上轉換 64-token " +"KV 頁；不相容或傳輸失敗會報錯。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Streaming and non-streaming requests require ``n=1``. SGLang decode uses the" +" first output token sampled by prefill. vLLM decode imports all but the " +"final prompt token, computes that token and samples the output. Engine " +"kernels can produce small numerical differences. vLLM decode requires at " +"least two prompt tokens." +msgstr "" +"串流與非串流請求均要求 ``n=1``。SGLang decode 使用 prefill 取樣的第一個輸出 token。vLLM decode " +"匯入最後一個 prompt token 之前的 KV，計算該 token 後取樣輸出。引擎核心可能產生小幅數值差異。vLLM decode 要求至少兩個" +" prompt token。" + +#: source/user_guide/pd_separation.rst +msgid "" +"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" +" or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" +" not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " +"NIXL comparisons use the same engine on both roles. MLX is unsupported." +msgstr "" +"跨引擎 P/D 不支援 CPU 回退、Xavier 歷史快取、logprobs、結構化取樣、LoRA、推測解碼、多模態輸入或混合注意力。省略 " +"``xavier_gpu_cache_bytes`` 或設為 ``0``，不要設定 ``xavier_cache_bytes``。worker " +"重新啟動後需要重新部署。原生 NIXL 對照在兩個角色上使用相同引擎。不支援 MLX。" diff --git a/doc/source/user_guide/pd_separation.rst b/doc/source/user_guide/pd_separation.rst index 4e3e2f8efa..ce8cf7b193 100644 --- a/doc/source/user_guide/pd_separation.rst +++ b/doc/source/user_guide/pd_separation.rst @@ -57,7 +57,7 @@ SGLang >= ``0.5.21`` accepts the same prefill/decode ``replica_config`` through SGLang Xavier P/D transfers KV directly between GPUs through Xavier's NIXL transport. Prefill and decode run concurrently using SGLang's native P/D lifecycle. Source and destination GPU slots remain owned until transfers complete; decode also receives the first token sampled by prefill. Streaming and non-streaming responses are supported. Transfer failures raise errors. -The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend and cross-engine P/D are not exposed by this integration. +The model limitations are described in :ref:`user_guide_vllm_enhancement`. Install ``xoscar[nixl]>=0.11.1`` in worker and model environments and use reachable worker addresses. Missing NIXL fails launch without CPU fallback. Use the ``xavier`` transport; SGLang's own Mooncake backend is not exposed by this integration. For cross-engine P/D, use the per-replica engine settings below. SGLang GPU P/D does not use ``xavier_cache_bytes`` or CPU HiCache. Retained Xavier GPU history is not yet supported; ``xavier_gpu_cache_bytes`` may be omitted or set to ``0``. Relaunch the deployment after a worker restart. Measure TTFT and throughput against ordinary replicas and native SGLang P/D for your workload. @@ -101,7 +101,44 @@ MLX Xavier stores immutable CPU pages in a shared actor on the supervisor and tr Each 64-token FP16 page uses ``2 * 64 * num_layers * num_kv_heads * head_dim * 2`` bytes (K and V, with two bytes per value). For example, 36 layers, 8 KV heads and a head dimension of 128 require 9 MiB per page; 512 MiB holds 56 pages, or 3584 prefix tokens. Round each prefix up to a whole page and size ``xavier_cache_bytes`` for the distinct pages reserved by all concurrent P/D requests. Ordinary shared-cache requests skip publication when their prefix exceeds the cache capacity or the 4096-page protocol limit. -Use the default ``xavier`` transport for MLX; native NIXL is unsupported. All workers must reach the supervisor's actor address. Relaunch after a worker restart. Two replicas on one Mac share its Metal GPU; measure your workload before expecting a throughput benefit. Cross-engine P/D, including NVIDIA-to-Mac handoff, is not yet supported. +Use the default ``xavier`` transport for MLX; native NIXL is unsupported. All workers must reach the supervisor's actor address. Relaunch after a worker restart. Two replicas on one Mac share its Metal GPU; measure your workload before expecting a throughput benefit. Cross-engine P/D involving MLX, including NVIDIA-to-Mac handoff, is not yet supported. + +Cross-engine vLLM/SGLang P/D +-------------------------------------------------------------------------------- + +Set ``model_engine`` and ``engine_config`` on each ``replica_config`` entry to select vLLM or SGLang independently for each role. Cross-engine P/D requires Xavier GPU transport. Engine options inherit launch arguments; ``engine_config`` overrides them per replica. + +.. code-block:: python + + client.launch_model( + model_uid="qwen-cross-pd", + model_name="qwen2.5-instruct", + model_size_in_billions="0_5", + model_format="pytorch", + quantization="none", + model_engine="vLLM", + replica=2, + replica_config=[ + { + "role": "prefill", "model_engine": "vLLM", + "engine_config": {"max_model_len": 8192}, + "devices": [{"worker_ip": "WORKER_ADDRESS", + "n_gpu": 1, "gpu_idx": [0]}], + }, + { + "role": "decode", "model_engine": "SGLang", + "engine_config": {"context_length": 8192}, + "devices": [{"worker_ip": "WORKER_ADDRESS", + "n_gpu": 1, "gpu_idx": [1]}], + }, + ], + ) + +Both workers require identical local weights, tokenizer assets and context limits. Supported models are unquantized FP16 full-attention Qwen2, Qwen3 and Llama text models with TP=PP=DP=1. Requires vLLM >=0.21.0, SGLang >=0.5.21 and ``xoscar[nixl]>=0.11.1``. Xavier checks contracts and prompt token IDs, converts 64-token KV pages on GPU and raises errors on incompatibility or transfer failure. + +Streaming and non-streaming requests require ``n=1``. SGLang decode uses the first output token sampled by prefill. vLLM decode imports all but the final prompt token, computes that token and samples the output. Engine kernels can produce small numerical differences. vLLM decode requires at least two prompt tokens. + +Cross-engine P/D supports no CPU fallback, retained Xavier history, logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native NIXL comparisons use the same engine on both roles. MLX is unsupported. Native NIXL backend ------------------- diff --git a/xinference/core/pd_model.py b/xinference/core/pd_model.py index 73672ec09f..a055da93ac 100644 --- a/xinference/core/pd_model.py +++ b/xinference/core/pd_model.py @@ -262,6 +262,21 @@ async def free_prefill_model_cache(self, request_id: str): address=handoff["address"], uid=handoff["uid"] ) if handoff.get("mode") == "gpu": + if handoff.get("heterogeneous"): + from ..model.llm.xavier.backends.torch.pd import ( + CrossEngineGPUActor, + ) + + try: + source = await ref.source(handoff["room"]) + except RuntimeError: + source = None + if source: + sender = await xo.actor_ref( + address=source["address"], + uid=f"{CrossEngineGPUActor.default_uid()}-{source['rank']}", + ) + await sender.abort(handoff["room"]) await ref.release(handoff["room"]) else: await ref.release_handoff(handoff["ticket"]) @@ -313,7 +328,33 @@ async def _infer(self, method, inputs, *args, **kwargs): if args and args[0] and args[0].get("n", 1) != 1: # Handoff leases cover one decoder, not parallel sampling children. raise ValueError("PD KV handoff currently requires n=1") - if self._model_engine == "sglang": + if self._model_engine == "heterogeneous": + config = (args[0] if args else {}) or {} + if any( + config.get(key) is not None and config.get(key) is not False + for key in ("logprobs", "prompt_logprobs") + ) or any( + config.get(key) + for key in ( + "top_logprobs", + "return_logprob", + "top_logprobs_num", + "guided_json", + "guided_regex", + "guided_choice", + "guided_grammar", + "guided_json_object", + "structured_outputs", + "json_schema", + "regex", + "ebnf", + "response_format", + ) + ): + raise ValueError( + "Cross-engine PD does not yet transfer structured sampling or logprobs" + ) + if self._model_engine in ("sglang", "heterogeneous"): return await self._infer_sglang(method, inputs, args, kwargs, request_id) prefill = self._prefill_policy.schedule() decode = self._decode_policy.schedule() @@ -435,6 +476,8 @@ async def _infer_sglang(self, method, inputs, args, kwargs, request_id): room=uuid.uuid4().int % (2**63 - 1) + 1, **bootstrap, ) + if self._model_engine == "heterogeneous": + handoff["heterogeneous"] = True transfer_key = "sglang_nixl" if native else "sglang_xavier" self._request_set.add(request_id) if not native: diff --git a/xinference/core/replica_config.py b/xinference/core/replica_config.py index ff51aa86b1..93557a3c32 100644 --- a/xinference/core/replica_config.py +++ b/xinference/core/replica_config.py @@ -24,7 +24,7 @@ cluster topology. """ -from typing import List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from .._compat import BaseModel, Field, validator from .utils import parse_replica_model_uid @@ -81,6 +81,8 @@ class ReplicaConfig(_PlacementConfigBase): replica_uid: Optional[str] = None role: Literal["hybrid", "prefill", "decode"] = "hybrid" devices: List[DeviceConfig] = Field(default_factory=list) + model_engine: Optional[str] = None + engine_config: Dict[str, Any] = Field(default_factory=dict) @classmethod def from_dict(cls, data: dict) -> "ReplicaConfig": @@ -182,13 +184,18 @@ def validate_pd_replica_configs( """Validate a complete P/D topology before allocating any runtime resources.""" roles = {cfg.role for cfg in configs or []} if not roles.intersection({"prefill", "decode"}): + if any(cfg.model_engine or cfg.engine_config for cfg in configs or []): + raise ValueError("Per-replica engine settings require explicit PD roles") return False - if (model_engine or "").lower() not in ("vllm", "sglang", "mlx") or ( - model_type or "LLM" - ).lower() != "llm": + engines = { + (cfg.model_engine or model_engine or "").lower() for cfg in configs or [] + } + if not engines <= {"vllm", "sglang", "mlx"} or (model_type or "LLM").lower() != "llm": raise ValueError( "PD separation requires model_type=LLM and model_engine=vLLM, SGLang or MLX" ) + if "mlx" in engines and len(engines) > 1: + raise ValueError("Cross-engine PD requires vLLM and SGLang replicas") if roles != {"prefill", "decode"}: raise ValueError( "PD separation requires both prefill and decode replicas, without hybrid replicas" diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index 157bb3bb7f..b0bcf40c4c 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3110,6 +3110,21 @@ async def launch_builtin_model( transport_backend = normalize_xavier_transport_backend(transport_backend) if transport_backend == "nixl" and not pd_enabled: raise ValueError("NIXL requires explicit prefill and decode replica roles") + replica_engines = [ + (cfg.model_engine or model_engine or "").lower() + for cfg in replica_config or [] + ] + heterogeneous_pd = pd_enabled and len(set(replica_engines)) > 1 + if ( + pd_enabled + and not heterogeneous_pd + and any(cfg.model_engine for cfg in replica_config or []) + ): + model_engine = next( + cfg.model_engine for cfg in replica_config or [] if cfg.model_engine + ) + if heterogeneous_pd and transport_backend != "xavier": + raise ValueError("Cross-engine PD requires Xavier GPU transport") # Xavier-related requested_xavier = bool(kwargs.pop("enable_xavier", False)) if ( @@ -3124,8 +3139,13 @@ async def launch_builtin_model( (requested_xavier or pd_enabled) and transport_backend == "xavier" and (model_engine or "").lower() == "mlx" + and not heterogeneous_pd ) - if (requested_xavier or pd_enabled) and (model_engine or "").lower() == "mlx": + if ( + (requested_xavier or pd_enabled) + and (model_engine or "").lower() == "mlx" + and not heterogeneous_pd + ): if transport_backend != "xavier": raise ValueError("MLX Xavier requires the xavier transport") if ( @@ -3143,6 +3163,7 @@ async def launch_builtin_model( and transport_backend == "xavier" and model_engine is not None and model_engine.lower() == "sglang" + and not heterogeneous_pd ) sglang_nixl = ( pd_enabled @@ -3185,15 +3206,16 @@ async def launch_builtin_model( and transport_backend == "xavier" and model_engine is not None and model_engine.lower() == "vllm" + and not heterogeneous_pd ) from ..model.llm.xavier.transport import validate_gpu_cache_budget gpu_cache_bytes = validate_gpu_cache_budget( kwargs.pop("xavier_gpu_cache_bytes", None), - enable_xavier or sglang_xavier and pd_enabled, + enable_xavier or sglang_xavier and pd_enabled or heterogeneous_pd, replica, ) - if sglang_xavier and gpu_cache_bytes not in (None, 0): + if (sglang_xavier or heterogeneous_pd) and gpu_cache_bytes not in (None, 0): raise ValueError( "SGLang Xavier GPU PD does not yet support retained GPU history" ) @@ -3302,6 +3324,25 @@ async def _launch_one_model( self._workers_launching[_addr] = self._workers_launching.get(_addr, 0) + 1 replica_kwargs = dict(kwargs) + replica_engine = model_engine + if pd_enabled: + assert replica_config is not None + cfg = replica_config[rank - 1] + replica_engine = cfg.model_engine or model_engine + replica_kwargs.update(cfg.engine_config) + if heterogeneous_pd: + assert replica_config is not None + from ..model.llm.xavier.transport import get_transport_host + + replica_kwargs["_xavier_cache_config"] = { + "address": self.address, + "uid": f"xavier-cache-{model_uid}", + "role": replica_config[rank - 1].role, + "rank": rank, + "host": get_transport_host(worker_ref.address), + "heterogeneous": True, + "gpu_cache_bytes": 0, + } if sglang_xavier and pd_enabled: assert replica_config is not None from ..model.llm.xavier.transport import get_transport_host @@ -3336,7 +3377,7 @@ async def _launch_one_model( model_size_in_billions=model_size_in_billions, model_format=model_format, quantization=quantization, - model_engine=model_engine, + model_engine=replica_engine, model_type=model_type, n_gpu=n_gpu if replica_n_gpu is None else replica_n_gpu, request_limits=request_limits, @@ -3380,6 +3421,17 @@ async def _launch_one_model( async def _launch_model(): nonlocal download_hub try: + if heterogeneous_pd: + from ..model.llm.sglang.xavier.directory import XavierPDDirectory + + cache_ref = await xo.create_actor( + XavierPDDirectory, + address=self.address, + uid=f"xavier-cache-{model_uid}", + ) + if not hasattr(self, "_xavier_cache_mapping"): + self._xavier_cache_mapping = {} + self._xavier_cache_mapping[model_uid] = cache_ref if sglang_xavier or mlx_xavier: if mlx_xavier: from ..model.llm.xavier.backends.bytes.storage import ( @@ -3672,7 +3724,9 @@ async def _launch_model(): PDModelActor, model_uid, transport_backend=transport_backend, - model_engine=model_engine, + model_engine=( + "heterogeneous" if heterogeneous_pd else model_engine + ), address=self.address, uid=f"{model_uid}-{PDModelActor.default_uid()}", ) diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index 406b883f16..f70ecd369b 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -745,3 +745,40 @@ async def test_failed_cache_actor_cleanup_emits_warning(launch_runtime, caplog): with caplog.at_level(logging.WARNING): await supervisor.terminate_model("pd") assert "Destroy Xavier cache failed for pd" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("engines", [("vLLM", "SGLang"), ("SGLang", "vLLM")]) +async def test_cross_engine_pd_launches_one_adapter_per_replica( + launch_runtime, engines +): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + for index, (cfg, engine) in enumerate(zip(kwargs["replica_config"], engines)): + cfg.model_engine = engine + cfg.engine_config = {"engine_option": index} + await supervisor.launch_builtin_model(**kwargs) + assert set(actors) == {"XavierPDDirectory", "PDModelActor"} + assert actors["PDModelActor"].constructor_kwargs["model_engine"] == "heterogeneous" + for index, worker in enumerate(workers): + launch = worker.launch_builtin_model.call_args.kwargs + assert launch["model_engine"] == engines[index] + assert launch["engine_option"] == index + assert launch["xavier_config"] is None + assert launch["_xavier_cache_config"]["heterogeneous"] is True + assert launch["_xavier_cache_config"]["role"] == ("prefill", "decode")[index] + worker.launch_rank0_model.assert_not_awaited() + worker.start_transfer_for_vllm.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_cross_engine_rejects_native_transport_before_launch(launch_runtime): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + kwargs["replica_config"][1].model_engine = "SGLang" + kwargs["transfer_backend_type"] = "nixl" + with pytest.raises(ValueError, match="Xavier GPU transport"): + await supervisor.launch_builtin_model(**kwargs) + assert not actors + for worker in workers: + worker.launch_builtin_model.assert_not_awaited() diff --git a/xinference/core/tests/test_pd_model.py b/xinference/core/tests/test_pd_model.py index a015b17231..b7deb65924 100644 --- a/xinference/core/tests/test_pd_model.py +++ b/xinference/core/tests/test_pd_model.py @@ -86,10 +86,15 @@ async def test_infer_preserves_decode_config(router, method): @pytest.mark.asyncio -@pytest.mark.parametrize("backend", ["xavier", "nixl"]) -async def test_sglang_starts_decode_while_prefill_holds_source_slots(router, backend): +@pytest.mark.parametrize( + "engine,backend", + [("sglang", "xavier"), ("sglang", "nixl"), ("heterogeneous", "xavier")], +) +async def test_sglang_starts_decode_while_prefill_holds_source_slots( + router, engine, backend +): actor, prefill, decode = router - actor._model_engine = "sglang" + actor._model_engine = engine actor._transport_backend = backend if backend == "nixl": actor._sglang_bootstrap["p"] = dict(host="producer", port=12345) @@ -115,6 +120,8 @@ async def d(*args, **kwargs): assert handoff == d_config["_pd_kv_transfer_params"][key] if backend == "nixl": assert handoff["host"] == "producer" and handoff["port"] == 12345 + if engine == "heterogeneous": + assert handoff["heterogeneous"] is True assert not actor._request_set and not actor._direct_transfers @@ -163,6 +170,72 @@ async def test_native_sglang_bootstrap_refreshed_on_replica_replacement(): assert actor._sglang_bootstrap["p"]["port"] == 23456 +@pytest.mark.asyncio +@pytest.mark.parametrize("source_published", [False, True]) +async def test_cross_engine_cancel_drains_source_before_directory_release( + router, monkeypatch, source_published +): + actor, prefill, decode = router + actor._model_engine = "heterogeneous" + calls = [] + directory, sender = AsyncMock(), AsyncMock() + directory.source.return_value = ( + dict(address="gpu-host", rank=1) if source_published else None + ) + + async def drain(room): + calls.append(("drain", room)) + + async def release(room): + calls.append(("release", room)) + + sender.abort.side_effect = drain + directory.release.side_effect = release + monkeypatch.setattr(xo, "actor_ref", AsyncMock(side_effect=[directory, sender])) + started = asyncio.Event() + + async def pending(*args, **kwargs): + started.set() + await asyncio.Event().wait() + + prefill.generate.side_effect = pending + decode.generate.side_effect = pending + task = asyncio.create_task(actor._infer("generate", "prompt", {}, request_id="r")) + await started.wait() + handoff = actor._direct_transfers["r"] + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + expected = [("drain", handoff["room"])] if source_published else [] + assert calls == expected + [("release", handoff["room"])] + prefill.abort_request.assert_awaited_once() + decode.abort_request.assert_awaited_once() + assert not actor._request_set and not actor._direct_transfers + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "config", + [ + {"logprobs": 0}, + {"prompt_logprobs": 0}, + {"guided_choice": ["yes", "no"]}, + {"json_schema": {"type": "object"}}, + {"return_logprob": True}, + ], +) +async def test_cross_engine_rejects_unsupported_sampling_before_scheduling( + router, config +): + actor, prefill, decode = router + actor._model_engine = "heterogeneous" + with pytest.raises(ValueError, match="structured sampling or logprobs"): + await actor._infer("generate", "prompt", config, request_id="r") + prefill.generate.assert_not_awaited() + decode.generate.assert_not_awaited() + assert not actor._request_set + + @pytest.mark.asyncio async def test_stream_cleanup_on_disconnect(router): actor, prefill, decode = router diff --git a/xinference/model/llm/sglang/xavier/config.py b/xinference/model/llm/sglang/xavier/config.py index 793af00a3b..17410d19e2 100644 --- a/xinference/model/llm/sglang/xavier/config.py +++ b/xinference/model/llm/sglang/xavier/config.py @@ -144,6 +144,12 @@ def configure_xavier(model_path: str, model_config: dict, cache_config: dict) -> logger.warning( "SGLang Xavier casts this BF16 checkpoint to FP16; weights and KV use FP16" ) + if cache_config.get("heterogeneous"): + from ...xavier.pd_contract import build_pd_contract + + if model_config["page_size"] != 64: + raise ValueError("Cross-engine Xavier PD requires 64-token pages") + contract = build_pd_contract(model_path, model_config.get("context_length")) model_config["dtype"] = "float16" if cache_config.get("role"): if model_config.get("enable_hierarchical_cache") or any( diff --git a/xinference/model/llm/sglang/xavier/directory.py b/xinference/model/llm/sglang/xavier/directory.py index 7b61bf5124..b16c811264 100644 --- a/xinference/model/llm/sglang/xavier/directory.py +++ b/xinference/model/llm/sglang/xavier/directory.py @@ -17,6 +17,7 @@ def __init__(self): self.rooms = {} self.completed_requests = 0 self.gpu_bytes = 0 + self.imported_tokens = 0 self.peers = {} async def __pre_destroy__(self): @@ -61,7 +62,7 @@ def _drop(self, room): state["source_ready"].set() state["complete_ready"].set() - def prepare(self, room, namespace, prompt_hash, role, rank=None, timeout=None): + def prepare(self, room, namespace, prompt_hash, role, rank=None, timeout=None, prompt_tokens=None): self._expire() if namespace != self.namespace or role not in ("prefill", "decode"): raise ValueError("Unregistered SGLang Xavier PD namespace or role") @@ -74,6 +75,7 @@ def prepare(self, room, namespace, prompt_hash, role, rank=None, timeout=None): raise RuntimeError("Too many active SGLang Xavier PD requests") state = self.rooms[room] = dict( prompt_hash=prompt_hash, + prompt_tokens=prompt_tokens, roles=[], source=None, completed=False, @@ -88,6 +90,10 @@ def prepare(self, room, namespace, prompt_hash, role, rank=None, timeout=None): raise ValueError("SGLang Xavier PD prompt mismatch or duplicate role") state["deadline"] = max(state["deadline"], time.monotonic() + timeout) state["timeout"] = max(state["timeout"], timeout) + if prompt_tokens is not None: + if state["prompt_tokens"] not in (None, prompt_tokens): + raise ValueError("Cross-engine Xavier prompt length differs") + state["prompt_tokens"] = prompt_tokens state["roles"].append(role) if rank is not None: state["ranks"][role] = rank @@ -124,7 +130,7 @@ def source(self, room): raise RuntimeError("SGLang Xavier PD handoff expired or was cancelled") return state["source"] - def complete(self, room, gpu_bytes): + def complete(self, room, gpu_bytes, imported_tokens=None): state = self.rooms.get(room) if state is None or set(state["roles"]) != {"prefill", "decode"}: raise RuntimeError("SGLang Xavier PD request is unavailable") @@ -132,6 +138,11 @@ def complete(self, room, gpu_bytes): self.completed_requests += 1 self.gpu_bytes += gpu_bytes state["deadline"] = time.monotonic() + state["timeout"] + self.imported_tokens += ( + imported_tokens + if imported_tokens is not None + else (state["prompt_tokens"] or 0) + ) state["completed"] = True state["complete_ready"].set() @@ -144,6 +155,13 @@ def renew_completed(self, room): state["deadline"] = time.monotonic() + state["timeout"] return True + def request_info(self, room): + self._expire() + state = self.rooms.get(room) + if state is None: + raise RuntimeError("Cross-engine Xavier handoff is unavailable") + return dict(prompt_tokens=state["prompt_tokens"]) + def check(self, room): self._expire() state = self.rooms.get(room) @@ -167,5 +185,6 @@ def get_stats(self): active_handoffs=len(self.rooms), completed_requests=self.completed_requests, gpu_bytes=self.gpu_bytes, + imported_tokens=self.imported_tokens, peers=self.peers, ) diff --git a/xinference/model/llm/sglang/xavier/gpu.py b/xinference/model/llm/sglang/xavier/gpu.py index 391c5b0421..b0552f8cca 100644 --- a/xinference/model/llm/sglang/xavier/gpu.py +++ b/xinference/model/llm/sglang/xavier/gpu.py @@ -418,6 +418,12 @@ async def _start(self, args, contract): aux_bytes=args.aux_item_lens, ) ) + actor_class = XavierGPUActor + if self.config.get("heterogeneous"): + from ...xavier.backends.torch.pd import CrossEngineGPUActor + + actor_class = CrossEngineGPUActor + namespace = contract.fingerprint directory = await xo.actor_ref( address=self.config["address"], uid=self.config["uid"] ) @@ -428,6 +434,16 @@ async def _start(self, args, contract): # Preserve the exporting wrappers until the importing process stops. # Export the engine allocations once, as the vLLM connector does. self._ipc_caches, self.aux = _buffers(args, contract) + if self.config.get("heterogeneous"): + self._ipc_caches = { + name: cache.view(torch.float16).reshape( + len(cache), + contract.block_size, + contract.num_kv_heads, + contract.head_dim, + ) + for name, cache in self._ipc_caches.items() + } descriptors = { name: reduce_tensor(cache)[1] for name, cache in self._ipc_caches.items() } @@ -445,7 +461,7 @@ async def _start(self, args, contract): ) await self.pool.start() self.actor = await xo.create_actor( - XavierGPUActor, + actor_class, SimpleNamespace(gpu_id=args.gpu_id, aux_item_lens=args.aux_item_lens), contract, directory, @@ -454,7 +470,7 @@ async def _start(self, args, contract): ipc_descriptors=descriptors, allocate_strategy=ProcessIndex(1), address=self.pool.external_address, - uid=f"{XavierGPUActor.default_uid()}-{self.config['rank']}", + uid=f"{actor_class.default_uid()}-{self.config['rank']}", ) await directory.register_peer(self.config["rank"], self.actor.address) diff --git a/xinference/model/llm/sglang/xavier/pd.py b/xinference/model/llm/sglang/xavier/pd.py index 637bb2a437..f367b4bb12 100644 --- a/xinference/model/llm/sglang/xavier/pd.py +++ b/xinference/model/llm/sglang/xavier/pd.py @@ -22,7 +22,7 @@ def __init__(self, cache_config: dict, tokenizer): self._directory_actor: Optional[xo.ActorRef] = None self._namespace: Optional[str] = None # Keep fixed-size digests only, scoped to this deployment/tokenizer. - self._prompt_hashes: OrderedDict[bytes, str] = OrderedDict() + self._prompt_hashes: OrderedDict[bytes, tuple[str, int]] = OrderedDict() self._lease_tasks: dict[int, asyncio.Task] = {} async def _call(self, method, *args, timeout=10): @@ -40,26 +40,31 @@ async def prepare(self, prompt: str, transfer: dict) -> dict: if not isinstance(handoff, dict) or handoff.get("mode") != "gpu": raise ValueError("Missing SGLang Xavier GPU bootstrap metadata") key = hashlib.sha256(prompt.encode()).digest() - prompt_hash = self._prompt_hashes.pop(key, None) - if prompt_hash is None: + cached = self._prompt_hashes.pop(key, None) + if cached is None: tokens = self.tokenizer.encode(prompt) prompt_hash = hashlib.sha256(json.dumps(tokens).encode()).hexdigest() - self._prompt_hashes[key] = prompt_hash + prompt_tokens = len(tokens) + else: + prompt_hash, prompt_tokens = cached + self._prompt_hashes[key] = (prompt_hash, prompt_tokens) if len(self._prompt_hashes) > 256: self._prompt_hashes.popitem(last=False) if self._namespace is None: self._namespace = (await self._call("get_stats"))["namespace"] if self._namespace is None: raise ValueError("Unregistered SGLang Xavier PD namespace") - await self._call( - "prepare", + arguments = [ handoff.get("room"), self._namespace, prompt_hash, self.role, self.config.get("rank"), transfer_timeout(), - ) + ] + if self.config.get("heterogeneous"): + arguments.append(prompt_tokens) + await self._call("prepare", *arguments) return handoff async def accept(self, prompt: str, transfer: dict) -> dict: diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py index 86dcf5dd4c..8ffcca5cd6 100644 --- a/xinference/model/llm/vllm/core.py +++ b/xinference/model/llm/vllm/core.py @@ -520,7 +520,9 @@ def __init__( self._active_request_ids: Set[str] = set() self.lora_modules = peft_model self.lora_requests: List[Any] = [] - self._xavier_config = None + self._xavier_config = cast(Dict[str, Any], model_config or {}).pop( + "_xavier_cache_config", None + ) self._nixl_config = cast(Dict[str, Any], model_config or {}).pop( "_nixl_config", None ) @@ -546,6 +548,12 @@ def __init__( self._init_weight_cache(model_config or {}) def set_xavier_config(self, value: Optional[Dict]): + if ( + value is None + and self._xavier_config + and self._xavier_config.get("heterogeneous") + ): + return self._xavier_config = value # type: ignore def set_worker_addresses(self, shard: int, worker_addresses: List[str]): @@ -757,6 +765,12 @@ def load(self): multiprocessing.set_start_method("fork", force=True) self._device_count = self._get_cuda_count() + if self._xavier_config and self._xavier_config.get("heterogeneous"): + from .xavier.cross_engine import configure_cross_engine + + configure_cross_engine( + self.model_path, self._model_config, self._xavier_config + ) self._model_config = self._sanitize_model_config(self._model_config) reasoning_content = self._model_config.pop("reasoning_content") enable_thinking = self._model_config.pop("enable_thinking", False) diff --git a/xinference/model/llm/vllm/xavier/cross_engine.py b/xinference/model/llm/vllm/xavier/cross_engine.py new file mode 100644 index 0000000000..71ae4df18f --- /dev/null +++ b/xinference/model/llm/vllm/xavier/cross_engine.py @@ -0,0 +1,276 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""vLLM V1 adapter for canonical Xavier GPU pages shared with SGLang.""" + +import asyncio +import math +import os +from dataclasses import dataclass, field +from types import SimpleNamespace + +import torch +import xoscar as xo +from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata + +from ...xavier.actor_loop import release_actor_loop +from ...xavier.backends.torch.pd import CrossEngineGPUActor, canonical_vllm_views +from ...xavier.contract import KVCacheContract +from ...xavier.pd_contract import build_pd_contract, prompt_digest +from ...xavier.transport import gpu_pool_options +from .v1_connector import XavierConnector + + +def configure_cross_engine(model_path, model_config, cache_config): + if model_config is None: + raise ValueError("Missing cross-engine vLLM model configuration") + if any( + model_config.get(name) + for name in ( + "hf_overrides", + "rope_scaling", + "rope_theta", + "lora_modules", + "speculative_config", + "kv_transfer_config", + "tokenizer", + "tokenizer_revision", + "model_loader_extra_config", + ) + ) or model_config.get("load_format", "auto") not in ("auto", "safetensors", "pt"): + raise ValueError( + "Cross-engine Xavier requires local weights without semantic overrides" + ) + if model_config.get("dtype", "auto") not in ( + None, + "auto", + "half", + "float16", + ) or model_config.get("kv_cache_dtype", "auto") not in ("auto", "float16"): + raise ValueError("Cross-engine Xavier PD requires FP16 KV") + if model_config.get("quantization") not in (None, "none"): + raise ValueError("Cross-engine Xavier PD requires unquantized weights") + if model_config.get("block_size", 64) != 64: + raise ValueError("Cross-engine Xavier PD requires 64-token pages") + contract = build_pd_contract(model_path, model_config.get("max_model_len")) + cache_config["contract"] = contract.to_dict() + model_config.update(dtype="float16", block_size=64) + # Prefix reuse within P remains native. D owns complete per-request pages. + if cache_config["role"] == "decode": + model_config["enable_prefix_caching"] = False + + +@dataclass +class CrossEngineMetadata(KVConnectorMetadata): + direct_sends: set[str] = field(default_factory=set) + direct_store: bool = False + loads: list[tuple[str, int, list[int]]] = field(default_factory=list) + + +class CrossEngineConnector(XavierConnector): + def __init__(self, vllm_config, role, kv_cache_config): + super().__init__(vllm_config, role, kv_cache_config) + self.contract = KVCacheContract.from_dict(self._xavier_config["contract"]) + if ( + self._block_size != self.contract.block_size + or vllm_config.model_config.dtype != torch.float16 + ): + raise ValueError("Cross-engine vLLM KV differs from the FP16 contract") + self._history_enabled = False + self._pool = None + self._directory = None + self._cross_requests = {} + self._cross_allocated = {} + self._prepared = set() + + async def directory(self): + if self._directory is None: + self._directory = await xo.actor_ref( + address=self._xavier_config["address"], uid=self._xavier_config["uid"] + ) + return self._directory + + async def _get_transfer_ref(self): + if self._transfer_ref is None: + directory = await self.directory() + peers = (await directory.get_stats())["peers"] + self._transfer_ref = await xo.actor_ref( + address=peers[self._rank], + uid=f"{CrossEngineGPUActor.default_uid()}-{self._rank}", + ) + return self._transfer_ref + + async def _ensure_gpu_cache_mapping(self): + from torch.multiprocessing.reductions import reduce_tensor + from xoscar.backends.allocate_strategy import ProcessIndex + + async with self._gpu_mapping_lock: + if not self._gpu_cache_mapped: + caches = canonical_vllm_views( + self._registered_kv_caches, self._num_cache_blocks, self.contract + ) + directory = await self.directory() + await directory.configure(self.contract.fingerprint) + options = gpu_pool_options(self._xavier_config["host"], os.environ) + self._pool = await xo.create_actor_pool( + options["external_address"], + n_process=1, + subprocess_start_method="spawn", + auto_recover=False, + ) + await self._pool.start() + descriptors = { + name: reduce_tensor(cache)[1] for name, cache in caches.items() + } + torch.cuda.synchronize() + self._transfer_ref = await xo.create_actor( + CrossEngineGPUActor, + SimpleNamespace( + gpu_id=torch.cuda.current_device(), aux_item_lens=[] + ), + self.contract, + directory, + self.contract.fingerprint, + self._rank, + ipc_descriptors=descriptors, + allocate_strategy=ProcessIndex(1), + address=self._pool.external_address, + uid=f"{CrossEngineGPUActor.default_uid()}-{self._rank}", + ) + await directory.register_peer(self._rank, self._transfer_ref.address) + self._gpu_cache_mapped = True + return self._transfer_ref + + def register_kv_caches(self, kv_caches): + super().register_kv_caches(kv_caches) + self._call(self._ensure_gpu_cache_mapping()) + + async def _prepare(self, request): + params = request.kv_transfer_params or {} + handoff = params.get("sglang_xavier") + if not isinstance(handoff, dict) or handoff.get("mode") != "gpu": + raise ValueError("Missing cross-engine Xavier GPU room") + room = handoff["room"] + if request.request_id not in self._prepared: + directory = await self.directory() + await directory.prepare( + room, + self.contract.fingerprint, + prompt_digest(request.prompt_token_ids), + self._xavier_config["role"], + rank=self._xavier_config["rank"], + prompt_tokens=len(request.prompt_token_ids), + ) + self._prepared.add(request.request_id) + return room + + def get_num_new_matched_tokens(self, request, num_computed_tokens): + if not self._is_consumer: + return 0, False + params = request.kv_transfer_params or {} + if params.get("do_remote_prefill") is False: + return 0, False + if len(request.prompt_token_ids) < 2: + raise ValueError( + "Cross-engine vLLM decode requires at least two prompt tokens" + ) + room = self._call(self._prepare(request)) + tokens = max(len(request.prompt_token_ids) - 1 - num_computed_tokens, 0) + if tokens: + self._cross_requests[request.request_id] = room + return tokens, bool(tokens) + + def update_state_after_alloc(self, request, blocks, num_external_tokens): + if request.request_id not in self._cross_requests or not num_external_tokens: + return + ids = blocks.get_block_ids() + self._cross_allocated[request.request_id] = ( + self._cross_requests[request.request_id], + list(ids[0]), + ) + request.kv_transfer_params["do_remote_prefill"] = False + + def build_connector_meta(self, scheduler_output): + meta = CrossEngineMetadata( + self._direct_sends, bool(scheduler_output.num_scheduled_tokens) + ) + self._direct_sends = set() + for request_id, (room, destinations) in self._cross_allocated.items(): + meta.loads.append((request_id, room, destinations)) + self._cross_allocated.clear() + return meta + + def start_load_kv(self, forward_context, **kwargs): + meta = self._get_connector_metadata() + self._direct_sends.update(meta.direct_sends) + for request_id, room, destinations in meta.loads: + + async def submit( + room=room, destinations=destinations, request_id=request_id + ): + actor = await self._ensure_gpu_cache_mapping() + task = asyncio.create_task(actor.receive(room, destinations, 0)) + self._gpu_load_jobs[task] = [request_id] + + self._call(submit()) + + def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs): + pass + + def wait_for_save(self): + if self._is_producer and self._get_connector_metadata().direct_store: + torch.cuda.synchronize() + + def request_finished(self, request, block_ids): + self._cross_requests.pop(request.request_id, None) + if self._is_producer and (request.kv_transfer_params or {}).get( + "do_remote_decode" + ): + from vllm.v1.request import RequestStatus + + if request.status == RequestStatus.FINISHED_ABORTED: + return False, None + room = self._call(self._prepare(request)) + count = math.ceil(len(request.prompt_token_ids) / self._block_size) + pages = block_ids[:count] + if len(pages) != count or not request.output_token_ids: + raise RuntimeError("Incomplete cross-engine vLLM prefill") + + async def register(): + actor = await self._get_transfer_ref() + await actor.register_prefill( + room, + request.request_id, + pages, + request.output_token_ids[0], + len(request.prompt_token_ids), + ) + + self._call(register()) + self._direct_sends.add(request.request_id) + self._prepared.discard(request.request_id) + return True, dict( + do_remote_prefill=True, + sglang_xavier=request.kv_transfer_params["sglang_xavier"], + ) + self._prepared.discard(request.request_id) + return False, None + + def request_finished_all_groups(self, request, block_ids): + return self.request_finished(request, list(block_ids[0])) + + def shutdown(self): + if self._loop is None: + return + + async def stop(): + if self._gpu_load_jobs: + await asyncio.gather(*self._gpu_load_jobs, return_exceptions=True) + if self._pool is not None: + await xo.destroy_actor(self._transfer_ref) + await self._pool.stop() + + try: + self._call(stop()) + finally: + release_actor_loop(self._loop) + self._loop = None diff --git a/xinference/model/llm/vllm/xavier/engine.py b/xinference/model/llm/vllm/xavier/engine.py index 851c8b0f3d..64fc9b6e9d 100644 --- a/xinference/model/llm/vllm/xavier/engine.py +++ b/xinference/model/llm/vllm/xavier/engine.py @@ -127,15 +127,24 @@ def _patch_v1_engine_args( ) engine_args.kv_transfer_config = KVTransferConfig( - kv_connector=XAVIER_CONNECTOR, - kv_connector_module_path=XAVIER_CONNECTOR_MODULE, + kv_connector=( + "CrossEngineConnector" + if xavier_config.get("heterogeneous") + else XAVIER_CONNECTOR + ), + kv_connector_module_path=( + "xinference.model.llm.vllm.xavier.cross_engine" + if xavier_config.get("heterogeneous") + else XAVIER_CONNECTOR_MODULE + ), engine_id=xavier_config.get("engine_id"), kv_role=kv_role, kv_rank=xavier_config.get("kv_rank", kv_rank), kv_parallel_size=xavier_config.get("kv_parallel_size", 2), kv_connector_extra_config=extra_config, kv_load_failure_policy=xavier_config.get( - "kv_load_failure_policy", "recompute" + "kv_load_failure_policy", + "fail" if xavier_config.get("heterogeneous") else "recompute", ), ) diff --git a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py new file mode 100644 index 0000000000..59cd3bbef5 --- /dev/null +++ b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py @@ -0,0 +1,102 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + + +@pytest.fixture +def cross_module(connector_module, monkeypatch): + monkeypatch.setitem( + sys.modules, "xinference.model.llm.vllm.xavier.v1_connector", connector_module + ) + name = "xinference.model.llm.vllm.xavier._test_cross_engine" + spec = importlib.util.spec_from_file_location( + name, Path(__file__).parents[1] / "cross_engine.py" + ) + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + return module + + +def decode(cross_module, tokens): + instance = cross_module.CrossEngineConnector.__new__( + cross_module.CrossEngineConnector + ) + instance._is_consumer = True + instance._is_producer = False + instance._cross_requests = {} + instance._cross_allocated = {} + instance._direct_sends = set() + instance._prepared = set() + instance._call = lambda coro: coro + instance._prepare = lambda request: 123 + request = SimpleNamespace( + request_id="r", + prompt_token_ids=tokens, + kv_transfer_params=dict(do_remote_prefill=True), + ) + return instance, request + + +def test_vllm_decode_reuses_all_but_final_prompt_token(cross_module): + instance, request = decode(cross_module, list(range(193))) + assert instance.get_num_new_matched_tokens(request, 0) == (192, True) + blocks = SimpleNamespace(get_block_ids=lambda: ([9, 5, 7, 8],)) + instance.update_state_after_alloc(request, blocks, 192) + meta = instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) + assert meta.loads == [("r", 123, [9, 5, 7, 8])] + assert request.kv_transfer_params["do_remote_prefill"] is False + assert instance.get_num_new_matched_tokens(request, 0) == (0, False) + # Preemption subsequently recomputes locally; the consumed room is single-use. + assert instance._cross_allocated == {} + + +def test_vllm_decode_rejects_zero_reuse_prompt(cross_module): + instance, request = decode(cross_module, [1]) + with pytest.raises(ValueError, match="two prompt tokens"): + instance.get_num_new_matched_tokens(request, 0) + + +@pytest.mark.asyncio +async def test_cross_prepare_checks_actual_token_ids_once(cross_module): + instance = cross_module.CrossEngineConnector.__new__( + cross_module.CrossEngineConnector + ) + instance._prepared = set() + instance.contract = SimpleNamespace(fingerprint="a" * 64) + instance._xavier_config = dict(role="decode") + directory = AsyncMock() + instance.directory = AsyncMock(return_value=directory) + request = SimpleNamespace( + request_id="r", + prompt_token_ids=[1, 2, 3], + kv_transfer_params={"sglang_xavier": dict(mode="gpu", room=9)}, + ) + assert await instance._prepare(request) == 9 + assert await instance._prepare(request) == 9 + directory.prepare.assert_awaited_once() + args = directory.prepare.call_args.args + assert args == (9, "a" * 64, cross_module.prompt_digest([1, 2, 3]), "decode", 3) + + +@pytest.mark.parametrize( + "option,value", + [ + ("quantization", "awq"), + ("dtype", "bfloat16"), + ("block_size", 16), + ("hf_overrides", {"rope_theta": 500000}), + ("tokenizer", "another"), + ], +) +def test_cross_engine_rejects_incompatible_runtime_options(cross_module, option, value): + with pytest.raises(ValueError): + cross_module.configure_cross_engine( + "unused", {option: value}, {"role": "decode"} + ) diff --git a/xinference/model/llm/xavier/backends/torch/pd.py b/xinference/model/llm/xavier/backends/torch/pd.py new file mode 100644 index 0000000000..9a30982f8b --- /dev/null +++ b/xinference/model/llm/xavier/backends/torch/pd.py @@ -0,0 +1,353 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""Canonical per-layer K/V pages, using Xavier's persistent GPU NIXL slabs.""" + +import asyncio +import os +import re +import struct +import time +from types import SimpleNamespace +from typing import Any + +import torch +import xoscar as xo + +from ....sglang.gc_lifecycle import InitializationGCFreeze +from .direct_handoff import DirectGPUTransfer +from .gpu_transfer import finish_before_cancel +from .snapshot import block_major_view + + +def sglang_first_token_payload(sizes, room, token, prompt_tokens): + # SGLang >=0.5.21 MetadataBuffers: output_ids, cached_tokens, logprobs, + # top-logprobs, speculative buffers, hidden states, bootstrap room. Logprobs, + # speculation, sampling masks and checksums are rejected for this protocol. + if len(sizes) != 10 or sizes[0] != 64 or sizes[1] != 64 or sizes[-1] != 64: + raise ValueError("Unsupported SGLang first-token metadata layout") + payload = [bytearray(size) for size in sizes] + struct.pack_into(" state["total"]: + raise ValueError("SGLang Xavier sent too many KV pages") + state["pending"].extend(pages) + # SGLang submits cached prefix pages separately from the final prefill + # page. Coalesce small chunks, keeping large prefill transfers pipelined. + if not final and len(state["pending"]) < self.chunk_capacity: + return + pages, state["pending"] = state["pending"], [] + ticket = f"{room}:{len(state['chunks'])}" + aux = None + if final: + aux = aux_payload + if aux is None: + raise ValueError("Missing SGLang Xavier first-token metadata") + self.transfer.register_direct(ticket, ticket, pages) + # Scheduler-owned source slots remain pinned until every chunk is read. + state["chunks"].append(dict(ticket=ticket, pages=pages, final=final, aux=aux)) + + def chunk(self, room, index): + state = self.rooms.get(room) + if state is None or state.get("aborted"): + raise RuntimeError("SGLang Xavier producer was cancelled") + return state["chunks"][index] if index < len(state["chunks"]) else None + + async def send_direct_gpu_v1(self, ticket, reads, remote_ref, slab_bytes): + return await self.transfer.run( + self.transfer.send_direct, ticket, reads, remote_ref, slab_bytes + ) + + def release_chunk(self, ticket): + self.transfer.release_direct(ticket) + room = int(ticket.split(":", 1)[0]) + state = self.rooms.get(room) + if state: + state["released"].add(ticket) + if state["sent"] == state["total"] and len(state["released"]) == len( + state["chunks"] + ): + state["completed"].set() + + async def release_remote_direct_gpu_v1(self, rank, ticket): + ref = await xo.actor_ref( + address=self._world_addresses[rank], uid=f"{self.default_uid()}-{rank}" + ) + await ref.release_chunk(ticket) + + def done(self, room): + self.transfer.poll_direct() + state = self.rooms[room] + return state["sent"] == state["total"] and len(state["released"]) == len( + state["chunks"] + ) + + async def wait_done(self, room): + state = self.rooms[room] + await state["completed"].wait() + if state.get("aborted"): + raise RuntimeError("SGLang Xavier producer was cancelled") + return self.done(room) + + def get_stats(self): + return dict( + self.transfer.metrics, + transfer_pid=os.getpid(), + active_rooms=len(self.rooms), + active_transfers=len(self.transfer.direct_requests), + ) + + async def _receive(self, room, destinations, aux_index): + deadline = time.monotonic() + 120 + source = None + while source is None: + source = await self.directory.source(room) + if time.monotonic() > deadline: + raise TimeoutError("SGLang Xavier producer bootstrap timed out") + if source is None: + await asyncio.sleep(0.005) + rank = source["rank"] + self._world_addresses[rank] = source["address"] + sender = await xo.actor_ref( + address=source["address"], uid=f"{self.default_uid()}-{rank}" + ) + offset, index, nbytes = 0, 0, 0 + while True: + chunk = await sender.chunk(room, index) + if chunk is None: + if time.monotonic() > deadline: + raise TimeoutError("SGLang Xavier producer KV transfer timed out") + await asyncio.sleep(0.001) + continue + pages = chunk["pages"] + targets = destinations[offset : offset + len(pages)] + if len(targets) != len(pages): + raise ValueError( + "SGLang Xavier source and destination page counts differ" + ) + mapping = dict(zip(pages, targets)) + if len(mapping) != len(pages): + raise ValueError("Duplicate SGLang Xavier source pages") + invalid = await self.transfer.run( + self.transfer.load_direct, + [{rank: {name: mapping for name in self.caches}}], + [chunk["ticket"]], + ) + if invalid: + raise RuntimeError("SGLang Xavier GPU handoff is unavailable") + offset += len(pages) + nbytes += len(pages) * sum( + cache[0].numel() * cache.element_size() + for cache in self.caches.values() + ) + index += 1 + if chunk["final"]: + if not self.args.aux_item_lens: + if offset != len(destinations): + raise ValueError("Cross-engine KV page counts differ") + stats = await self.directory.request_info(room) + await self.directory.complete( + room, nbytes, max(stats["prompt_tokens"] - 1, 0) + ) + await self.directory.release(room, "decode") + return set() + if isinstance(chunk["aux"], dict): + chunk["aux"] = sglang_first_token_payload( + self.args.aux_item_lens, + room, + chunk["aux"]["first_token"], + chunk["aux"]["prompt_tokens"], + ) + if offset != len(destinations) or len(chunk["aux"]) != len( + self.args.aux_item_lens + ): + raise ValueError( + "SGLang Xavier KV or auxiliary metadata geometry differs" + ) + for size, payload in zip(self.args.aux_item_lens, chunk["aux"]): + if len(payload) != size: + raise ValueError( + "SGLang Xavier auxiliary metadata size differs" + ) + # Raw CPU metadata addresses belong to the engine process. + # Its manager publishes the first-token/room marker after these + # GPU writes drain, before native decode can commit the slots. + return nbytes, chunk["aux"] + + async def receive(self, room, destinations, aux_index): + task = self.tasks[room] = asyncio.create_task( + self._receive(room, destinations, aux_index) + ) + try: + return await finish_before_cancel(task) + finally: + self.tasks.pop(room, None) + + async def abort(self, room): + task = self.tasks.get(room) + if task is not None: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + # Do not return engine slots while a peer's gather is still reading them. + async with self.transfer.send_lock: + state = self.rooms.get(room) + if state: + state["aborted"] = True + for chunk in state["chunks"]: + self.transfer.release_direct(chunk["ticket"]) + state["completed"].set() + + def clear(self, room): + self.rooms.pop(room, None) diff --git a/xinference/model/llm/xavier/pd_contract.py b/xinference/model/llm/xavier/pd_contract.py new file mode 100644 index 0000000000..faef3fa309 --- /dev/null +++ b/xinference/model/llm/xavier/pd_contract.py @@ -0,0 +1,76 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +"""Common model identity and token boundaries for cross-engine GPU PD.""" + +import hashlib +import json +from pathlib import Path + +from .contract import KVCacheContract, fingerprint_files, fingerprint_metadata + + +def prompt_digest(tokens: list[int]) -> str: + return hashlib.sha256(json.dumps(tokens).encode()).hexdigest() + + +def build_pd_contract( + model_path: str, context_length: int | None = None +) -> KVCacheContract: + path = Path(model_path) + config = json.loads((path / "config.json").read_text()) + if ( + config.get("model_type") not in ("qwen2", "qwen3", "llama") + or config.get("use_sliding_window") + or config.get("vision_config") + or config.get("quantization_config") + or config.get("auto_map") + or any(kind != "full_attention" for kind in config.get("layer_types", [])) + ): + raise ValueError( + "Cross-engine Xavier PD requires full-attention Qwen2, Qwen3 or Llama text weights" + ) + weights = sorted(path.glob("*.safetensors")) or sorted( + path.glob("pytorch_model*.bin") + ) + tokenizer = [ + path / name + for name in ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "tokenizer.model", + "spiece.model", + ) + if (path / name).is_file() + ] + if not weights or not any( + file.name in ("tokenizer.json", "tokenizer.model", "spiece.model", "vocab.json") + for file in tokenizer + ): + raise ValueError( + "Cross-engine Xavier PD requires local weights and tokenizer assets" + ) + return KVCacheContract( + weights_fingerprint=fingerprint_files({file.name: file for file in weights}), + tokenizer_fingerprint=fingerprint_files( + {file.name: file for file in tokenizer} + ), + attention_fingerprint=fingerprint_metadata(config), + position_fingerprint=fingerprint_metadata( + { + "config": config, + "context_length": context_length + or config.get("max_position_embeddings"), + } + ), + num_layers=config["num_hidden_layers"], + num_kv_heads=config.get("num_key_value_heads", config["num_attention_heads"]), + head_dim=config.get( + "head_dim", config["hidden_size"] // config["num_attention_heads"] + ), + block_size=64, + logical_dtype="float16", + ) diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py new file mode 100644 index 0000000000..c862434bd0 --- /dev/null +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -0,0 +1,108 @@ +# Copyright 2022-2026 Xinference Holdings Pte. Ltd +# Licensed under the Apache License, Version 2.0. +import json +import struct +from dataclasses import replace + +import pytest +import torch + +from ...sglang.xavier.directory import XavierPDDirectory +from ..backends.torch.pd import canonical_vllm_views, sglang_first_token_payload +from ..pd_contract import build_pd_contract, prompt_digest + + +@pytest.mark.parametrize("layout", ["2NTHD", "N2THD", "BHN2D", "BHN2D_NHD"]) +def test_vllm_canonical_views_preserve_engine_storage(kv_contract, layout): + contract = replace(kv_contract, num_layers=2) + tensors = { + f"model.layers.{i}.self_attn.attn": torch.arange( + 7 * 2 * 4 * 2 * 4, dtype=torch.float16 + ).reshape(7, 2, 4, 2, 4) + + i * 1000 + for i in range(2) + } + original = {name: tensor.clone() for name, tensor in tensors.items()} + if layout == "2NTHD": + tensors = {name: tensor.movedim(1, 0) for name, tensor in tensors.items()} + if layout.startswith("BHN2D"): + tensors = { + name: tensor.permute(0, 3, 2, 1, 4).reshape(7, 2, 4, 8) + for name, tensor in tensors.items() + } + if layout == "BHN2D_NHD": + tensors = { + name: tensor.transpose(1, 2).contiguous().transpose(1, 2) + for name, tensor in tensors.items() + } + views = canonical_vllm_views(tensors, 7, contract) + for kind in range(2): + for index in range(2): + name = f"model.layers.{index}.self_attn.attn" + view = views[str(index + kind * 2)] + assert torch.equal(view, original[name][:, kind]) + view[3, 2] = -1 + native = tensors[name] if layout == "N2THD" else tensors[name].movedim(1, 0) + if layout.startswith("BHN2D"): + assert (tensors[name][3, :, 2, kind * 4 : (kind + 1) * 4] == -1).all() + else: + assert (native[3, kind, 2] == -1).all() + + +def test_canonical_views_reject_missing_layer_and_dtype(kv_contract): + contract = replace(kv_contract, num_layers=1) + with pytest.raises(ValueError, match="Incomplete"): + canonical_vllm_views({}, 7, contract) + with pytest.raises(ValueError, match="layout"): + canonical_vllm_views( + {"model.layers.0.self_attn.attn": torch.zeros(7, 2, 4, 2, 4)}, 7, contract + ) + + +def test_first_token_metadata_and_room_marker(): + sizes = [64, 64, 64, 64, 512, 512, 64, 128, 1024, 64] + payload = sglang_first_token_payload(sizes, 123, 42, 193) + assert list(map(len, payload)) == sizes + assert struct.unpack_from("= 2 + contract = replace(kv_contract, num_layers=2, block_size=64, head_dim=8) + caches, views = [], [] + for gpu in (0, 1): + engine = producer if gpu == 0 else ("sglang" if producer == "vllm" else "vllm") + if engine == "vllm": + # vLLM 0.28 logical [B,H,T,2D], stored physically in NHD order. + native = { + f"model.layers.{layer}.self_attn.attn": torch.zeros( + 7, 64, 2, 16, device=f"cuda:{gpu}", dtype=torch.float16 + ).transpose(1, 2) + for layer in range(2) + } + canonical = canonical_vllm_views(native, 7, contract) + else: + native = canonical = { + str(layer): torch.zeros( + 7, 64, 2, 8, device=f"cuda:{gpu}", dtype=torch.float16 + ) + for layer in range(4) + } + caches.append(native) + views.append(canonical) + for layer, cache in enumerate(views[0].values()): + cache.copy_( + (torch.arange(cache.numel(), device="cuda:0") % 257).reshape(cache.shape) + + layer * 300 + ) + for gpu in (0, 1): + torch.cuda.synchronize(gpu) + options = gpu_pool_options("127.0.0.1", os.environ) + pool = await xo.create_actor_pool( + options["external_address"], n_process=2, subprocess_start_method="spawn" + ) + actors = [] + try: + await pool.start() + directory = await xo.create_actor( + XavierPDDirectory, address=pool.external_address + ) + await directory.configure(contract.fingerprint) + for gpu in (0, 1): + actor = await xo.create_actor( + CrossEngineGPUActor, + SimpleNamespace(gpu_id=gpu, aux_item_lens=[]), + contract, + directory, + contract.fingerprint, + gpu + 1, + ipc_descriptors={ + key: reduce_tensor(value)[1] for key, value in views[gpu].items() + }, + address=pool.external_address, + allocate_strategy=ProcessIndex(gpu + 1), + uid=f"{CrossEngineGPUActor.default_uid()}-{gpu + 1}", + ) + actors.append(actor) + room, source, destination = 123, [5, 1, 3], [2, 6, 4] + for role in ("prefill", "decode"): + await directory.prepare( + room, contract.fingerprint, prompt_digest(list(range(129))), role, 129 + ) + if producer == "vllm": + await actors[0].register_prefill(room, "request", source, 42, 129) + else: + await actors[0].open(room) + await actors[0].init(room, len(source), 0) + await actors[0].add_chunk(room, source[:1]) + await actors[0].add_chunk(room, source[1:], [b"first-token"]) + assert await actors[1].receive(room, destination, 0) == set() + assert await actors[0].poll_direct_gpu_v1() == { + "request" if producer == "vllm" else f"{room}:0" + } + for key, original in views[0].items(): + assert torch.equal(original[source].cpu(), views[1][key][destination].cpu()) + stats = await directory.get_stats() + assert stats["imported_tokens"] == 128 and stats["active_handoffs"] == 0 + for actor in actors: + stats = await actor.get_stats() + assert stats["cpu_batches"] == stats["active_transfers"] == 0 + finally: + for actor in actors: + await xo.destroy_actor(actor) + await pool.stop() From 49cd8e09c0bae9f717beb500e0ad0ae7a0b41cb9 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 00:55:27 +0800 Subject: [PATCH 2/8] fix: contain cancelled cross-engine KV loads to their request --- benchmark/benchmark_cross_engine_pd.py | 88 ++++++++++++------- .../model/llm/sglang/tests/test_xavier.py | 5 ++ xinference/model/llm/sglang/xavier/config.py | 4 + .../model/llm/vllm/xavier/cross_engine.py | 22 ++++- .../llm/vllm/xavier/test/test_cross_engine.py | 31 +++++++ 5 files changed, 114 insertions(+), 36 deletions(-) diff --git a/benchmark/benchmark_cross_engine_pd.py b/benchmark/benchmark_cross_engine_pd.py index 5e9f967650..4961ef9c67 100644 --- a/benchmark/benchmark_cross_engine_pd.py +++ b/benchmark/benchmark_cross_engine_pd.py @@ -184,43 +184,55 @@ async def requests_and_measure(endpoint, uid, root, worker, mode, args): peer["active_transfers"] == 0 for peer in results["after_cancel"]["peers"].values() ) - # Cancel a prepared room while its long prefill is still running. - request_id = uuid.uuid4().hex - body = bodies()[3] - body["messages"][0]["content"] = ( - "A new uncached prefill. " * 1000 + "Explain evaporation." - ) - body["max_tokens"] = 1024 - body["extra_body"]["request_id"] = request_id - pending = asyncio.create_task( - api.chat.completions.create(model=uid, **body) - ) - directory = await xo.actor_ref(address=worker, uid="xavier-cache-" + uid) - for _ in range(1000): - if (await directory.get_stats())["active_handoffs"]: - break + results["early_cancels"] = [] + for attempt in range(3): + # Cancel a prepared room while its long prefill is still running. + request_id = uuid.uuid4().hex + body = bodies()[3] + body["messages"][0]["content"] = ( + f"Trial {attempt}. " + + "A new uncached prefill. " * 1000 + + "Explain evaporation." + ) + body["max_tokens"] = 1024 + body["extra_body"]["request_id"] = request_id + pending = asyncio.create_task( + api.chat.completions.create(model=uid, **body) + ) + directory = await xo.actor_ref( + address=worker, uid="xavier-cache-" + uid + ) + for _ in range(1000): + if (await directory.get_stats())["active_handoffs"]: + break + assert ( + not pending.done() + ), "Request finished before the cancellation probe" + await asyncio.sleep(0.001) + else: + raise AssertionError( + "Cancellation probe did not reach a prepared room" + ) + results["early_cancel"] = await asyncio.to_thread( + Client(endpoint).abort_request, uid, request_id + ) + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await asyncio.sleep(1) + await api.chat.completions.create(model=uid, **bodies()[0]) + await asyncio.sleep(0.1) + results["after_early_cancel"] = await snapshot(worker, uid, True) assert ( - not pending.done() - ), "Request finished before the cancellation probe" - await asyncio.sleep(0.001) - else: - raise AssertionError("Cancellation probe did not reach a prepared room") - results["early_cancel"] = await asyncio.to_thread( - Client(endpoint).abort_request, uid, request_id - ) - pending.cancel() - await asyncio.gather(pending, return_exceptions=True) - await asyncio.sleep(1) - await api.chat.completions.create(model=uid, **bodies()[0]) - await asyncio.sleep(0.1) - results["after_early_cancel"] = await snapshot(worker, uid, True) - assert results["after_early_cancel"]["directory"]["active_handoffs"] == 0 - assert all( - peer["active_transfers"] == 0 - for peer in results["after_early_cancel"]["peers"].values() - ) + results["after_early_cancel"]["directory"]["active_handoffs"] == 0 + ) + assert all( + peer["active_transfers"] == 0 + for peer in results["after_early_cancel"]["peers"].values() + ) + results["early_cancels"].append(results["after_early_cancel"]) finally: await api.close() + (root / "partial-results.json").write_text(json.dumps(results, indent=2)) (root / "results.json").write_text(json.dumps(results, indent=2)) return results @@ -350,6 +362,14 @@ def run(args): flush=True, ) finally: + (root / "isolation.json").write_text( + json.dumps( + dict( + samples=isolation.samples, interference=isolation.interference + ), + indent=2, + ) + ) stop_server(proc) diff --git a/xinference/model/llm/sglang/tests/test_xavier.py b/xinference/model/llm/sglang/tests/test_xavier.py index 4fbdf59764..5c38cff7f8 100644 --- a/xinference/model/llm/sglang/tests/test_xavier.py +++ b/xinference/model/llm/sglang/tests/test_xavier.py @@ -288,6 +288,11 @@ async def stats(): assert storage._rpc_loop.is_closed() +def test_cross_engine_cannot_select_unfingerprinted_pt_weights(model_path): + with pytest.raises(ValueError, match="automatic or safetensors"): + configure_xavier(model_path, {"load_format": "pt"}, {"heterogeneous": True}) + + def test_launch_configuration_consumes_adapter_options(model_path): config = {"tp_size": 1, "page_size": 4} configure_xavier(model_path, config, {"address": "worker:1234", "uid": "cache"}) diff --git a/xinference/model/llm/sglang/xavier/config.py b/xinference/model/llm/sglang/xavier/config.py index 17410d19e2..fb9f8b8264 100644 --- a/xinference/model/llm/sglang/xavier/config.py +++ b/xinference/model/llm/sglang/xavier/config.py @@ -147,6 +147,10 @@ def configure_xavier(model_path: str, model_config: dict, cache_config: dict) -> if cache_config.get("heterogeneous"): from ...xavier.pd_contract import build_pd_contract + if model_config.get("load_format", "auto") not in ("auto", "safetensors"): + raise ValueError( + "Cross-engine Xavier requires standard automatic or safetensors loading" + ) if model_config["page_size"] != 64: raise ValueError("Cross-engine Xavier PD requires 64-token pages") contract = build_pd_contract(model_path, model_config.get("context_length")) diff --git a/xinference/model/llm/vllm/xavier/cross_engine.py b/xinference/model/llm/vllm/xavier/cross_engine.py index 71ae4df18f..e3da8a63c9 100644 --- a/xinference/model/llm/vllm/xavier/cross_engine.py +++ b/xinference/model/llm/vllm/xavier/cross_engine.py @@ -3,6 +3,7 @@ """vLLM V1 adapter for canonical Xavier GPU pages shared with SGLang.""" import asyncio +import logging import math import os from dataclasses import dataclass, field @@ -19,6 +20,8 @@ from ...xavier.transport import gpu_pool_options from .v1_connector import XavierConnector +logger = logging.getLogger(__name__) + def configure_cross_engine(model_path, model_config, cache_config): if model_config is None: @@ -36,7 +39,7 @@ def configure_cross_engine(model_path, model_config, cache_config): "tokenizer_revision", "model_loader_extra_config", ) - ) or model_config.get("load_format", "auto") not in ("auto", "safetensors", "pt"): + ) or model_config.get("load_format", "auto") not in ("auto", "safetensors"): raise ValueError( "Cross-engine Xavier requires local weights without semantic overrides" ) @@ -208,11 +211,26 @@ async def submit( room=room, destinations=destinations, request_id=request_id ): actor = await self._ensure_gpu_cache_mapping() - task = asyncio.create_task(actor.receive(room, destinations, 0)) + task = asyncio.create_task( + self._receive_pages(actor, room, destinations) + ) self._gpu_load_jobs[task] = [request_id] self._call(submit()) + async def _receive_pages(self, actor, room, destinations): + try: + return await actor.receive(room, destinations, 0) + except Exception: + # The transfer actor fences writes before reporting an error. Return + # load errors through vLLM's per-request callback, including after an + # abort removes the room. Raising from get_finished kills EngineCore. + # kv_load_failure_policy=fail prevents fallback to local prefill. + logger.warning( + "Cross-engine KV load failed for room %s", room, exc_info=True + ) + return set(destinations) + def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs): pass diff --git a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py index 59cd3bbef5..cfefd88f70 100644 --- a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py +++ b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py @@ -1,5 +1,6 @@ # Copyright 2022-2026 Xinference Holdings Pte. Ltd # Licensed under the Apache License, Version 2.0. +import asyncio import importlib.util import sys from pathlib import Path @@ -33,6 +34,7 @@ def decode(cross_module, tokens): instance._cross_requests = {} instance._cross_allocated = {} instance._direct_sends = set() + instance._direct_handoff = True instance._prepared = set() instance._call = lambda coro: coro instance._prepare = lambda request: 123 @@ -93,6 +95,7 @@ async def test_cross_prepare_checks_actual_token_ids_once(cross_module): ("block_size", 16), ("hf_overrides", {"rope_theta": 500000}), ("tokenizer", "another"), + ("load_format", "pt"), ], ) def test_cross_engine_rejects_incompatible_runtime_options(cross_module, option, value): @@ -100,3 +103,31 @@ def test_cross_engine_rejects_incompatible_runtime_options(cross_module, option, cross_module.configure_cross_engine( "unused", {option: value}, {"role": "decode"} ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error", [RuntimeError("room cancelled"), TimeoutError("transfer")] +) +async def test_failed_load_reports_all_blocks_without_crashing_vllm( + cross_module, error +): + instance, _ = decode(cross_module, list(range(193))) + instance._loop = object() + instance._invalid_block_ids = set() + instance._call = lambda coro: coro.close() + actor = AsyncMock() + actor.receive.side_effect = error + task = asyncio.create_task(instance._receive_pages(actor, 123, [9, 5, 7, 8])) + instance._gpu_load_jobs = {task: ["r"]} + await task + assert instance.get_finished(set()) == (set(), {"r"}) + assert instance.get_block_ids_with_load_errors() == {9, 5, 7, 8} + assert not instance._gpu_load_jobs + actor.receive.side_effect = None + actor.receive.return_value = set() + task = asyncio.create_task(instance._receive_pages(actor, 124, [1, 2])) + instance._gpu_load_jobs = {task: ["next"]} + await task + assert instance.get_finished(set()) == (set(), {"next"}) + assert instance.get_block_ids_with_load_errors() == set() From a2761bd2bcdadc49a98a32d932e1cd224fe5c8b9 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 01:02:43 +0800 Subject: [PATCH 3/8] fix: handle late cross-engine bootstrap cancellation in SGLang --- xinference/model/llm/sglang/xavier/gpu.py | 4 +++ .../model/llm/xavier/backends/torch/pd.py | 14 ++++++++-- .../llm/xavier/tests/test_pd_cross_engine.py | 28 ++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/xinference/model/llm/sglang/xavier/gpu.py b/xinference/model/llm/sglang/xavier/gpu.py index b0552f8cca..6d5cab0f32 100644 --- a/xinference/model/llm/sglang/xavier/gpu.py +++ b/xinference/model/llm/sglang/xavier/gpu.py @@ -584,6 +584,8 @@ async def run(): return self._operation def init(self, num_kv_indices, aux_index=None): + if self.aborted: + return self.total, self.sent, self.aux_index = num_kv_indices, 0, aux_index initialized = self._enqueue("init", self.room, num_kv_indices, aux_index) @@ -601,6 +603,8 @@ async def wait_done(): self.inited = True def send(self, kv_indices, state_indices=None, num_kv_tokens=None): + if self.aborted: + return if state_indices: raise ValueError( "SGLang Xavier does not transfer auxiliary attention state" diff --git a/xinference/model/llm/xavier/backends/torch/pd.py b/xinference/model/llm/xavier/backends/torch/pd.py index 9a30982f8b..d8885badf9 100644 --- a/xinference/model/llm/xavier/backends/torch/pd.py +++ b/xinference/model/llm/xavier/backends/torch/pd.py @@ -143,9 +143,13 @@ async def open(self, room): sent=0, completed=asyncio.Event(), ) - await self.directory.publish_source( - room, dict(address=self.address, rank=self.rank) - ) + try: + await self.directory.publish_source( + room, dict(address=self.address, rank=self.rank) + ) + except BaseException: + self.rooms.pop(room, None) + raise async def register_prefill( self, room, request_id, pages, first_token, prompt_tokens @@ -175,10 +179,14 @@ async def poll_direct_gpu_v1(self): def init(self, room, count, aux_index): state = self.rooms[room] + if state.get("aborted"): + raise RuntimeError("Cross-engine producer was cancelled") state.update(total=count, aux_index=aux_index) async def add_chunk(self, room, pages, aux_payload: list[bytes] | None = None): state = self.rooms[room] + if state.get("aborted"): + raise RuntimeError("Cross-engine producer was cancelled") state["sent"] += len(pages) final = state["sent"] == state["total"] if state["sent"] > state["total"]: diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py index c862434bd0..ba36ddb3e2 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -3,12 +3,17 @@ import json import struct from dataclasses import replace +from unittest.mock import AsyncMock import pytest import torch from ...sglang.xavier.directory import XavierPDDirectory -from ..backends.torch.pd import canonical_vllm_views, sglang_first_token_payload +from ..backends.torch.pd import ( + CrossEngineGPUActor, + canonical_vllm_views, + sglang_first_token_payload, +) from ..pd_contract import build_pd_contract, prompt_digest @@ -106,3 +111,24 @@ def test_contract_fingerprints_actual_assets_and_context(tmp_path): (tmp_path / "model.safetensors").write_bytes(b"changed weights") with pytest.raises(ValueError, match="weights_fingerprint"): initial.require_match(build_pd_contract(str(tmp_path))) + + +@pytest.mark.asyncio +async def test_late_source_publication_does_not_leak_a_room(): + directory = AsyncMock() + directory.publish_source.side_effect = ValueError("cancelled room") + actor = CrossEngineGPUActor(None, None, directory, "namespace", 1) + with pytest.raises(ValueError, match="cancelled room"): + await actor.open(123) + assert not actor.rooms + + +@pytest.mark.asyncio +async def test_aborted_source_cannot_register_late_gpu_chunks(): + actor = CrossEngineGPUActor(None, None, None, "namespace", 1) + actor.rooms[123] = {"aborted": True} + with pytest.raises(RuntimeError, match="cancelled"): + actor.init(123, 3, 0) + with pytest.raises(RuntimeError, match="cancelled"): + await actor.add_chunk(123, [1, 2, 3]) + assert actor.rooms[123] == {"aborted": True} From a473ec835cc613f0bbfbfdaf6a5e5dea4a3eccef Mon Sep 17 00:00:00 2001 From: qinxuye Date: Tue, 6 Oct 2026 01:12:03 +0800 Subject: [PATCH 4/8] fix: release cross-engine rooms after bootstrap aborts --- benchmark/benchmark_cross_engine_pd.py | 19 ++++++++++++++++--- .../model/llm/xavier/backends/torch/pd.py | 4 ++++ .../llm/xavier/tests/test_pd_cross_engine.py | 17 ++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/benchmark/benchmark_cross_engine_pd.py b/benchmark/benchmark_cross_engine_pd.py index 4961ef9c67..1fe244a448 100644 --- a/benchmark/benchmark_cross_engine_pd.py +++ b/benchmark/benchmark_cross_engine_pd.py @@ -164,7 +164,10 @@ async def requests_and_measure(endpoint, uid, root, worker, mode, args): for peer in results["transfers"]["peers"].values() ) assert all( - peer["active_transfers"] == 0 + all( + peer[key] == 0 + for key in ("active_rooms", "active_transfers", "active_receives") + ) for peer in results["transfers"]["peers"].values() ) # Disconnect after the first streaming token, then verify fresh work. @@ -181,7 +184,10 @@ async def requests_and_measure(endpoint, uid, root, worker, mode, args): results["after_cancel"] = await snapshot(worker, uid, True) assert results["after_cancel"]["directory"]["active_handoffs"] == 0 assert all( - peer["active_transfers"] == 0 + all( + peer[key] == 0 + for key in ("active_rooms", "active_transfers", "active_receives") + ) for peer in results["after_cancel"]["peers"].values() ) results["early_cancels"] = [] @@ -226,7 +232,14 @@ async def requests_and_measure(endpoint, uid, root, worker, mode, args): results["after_early_cancel"]["directory"]["active_handoffs"] == 0 ) assert all( - peer["active_transfers"] == 0 + all( + peer[key] == 0 + for key in ( + "active_rooms", + "active_transfers", + "active_receives", + ) + ) for peer in results["after_early_cancel"]["peers"].values() ) results["early_cancels"].append(results["after_early_cancel"]) diff --git a/xinference/model/llm/xavier/backends/torch/pd.py b/xinference/model/llm/xavier/backends/torch/pd.py index d8885badf9..b9b5d0fc74 100644 --- a/xinference/model/llm/xavier/backends/torch/pd.py +++ b/xinference/model/llm/xavier/backends/torch/pd.py @@ -255,6 +255,7 @@ def get_stats(self): transfer_pid=os.getpid(), active_rooms=len(self.rooms), active_transfers=len(self.transfer.direct_requests), + active_receives=len(self.tasks), ) async def _receive(self, room, destinations, aux_index): @@ -356,6 +357,9 @@ async def abort(self, room): for chunk in state["chunks"]: self.transfer.release_direct(chunk["ticket"]) state["completed"].set() + # Native bootstrap aborts need not call sender.clear(). Waiters + # already hold the state and will observe its aborted marker. + self.rooms.pop(room, None) def clear(self, room): self.rooms.pop(room, None) diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py index ba36ddb3e2..0540a66c7d 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -1,9 +1,11 @@ # Copyright 2022-2026 Xinference Holdings Pte. Ltd # Licensed under the Apache License, Version 2.0. +import asyncio import json import struct from dataclasses import replace -from unittest.mock import AsyncMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock import pytest import torch @@ -132,3 +134,16 @@ async def test_aborted_source_cannot_register_late_gpu_chunks(): with pytest.raises(RuntimeError, match="cancelled"): await actor.add_chunk(123, [1, 2, 3]) assert actor.rooms[123] == {"aborted": True} + + +@pytest.mark.asyncio +async def test_bootstrap_abort_clears_room_and_wakes_existing_waiter(): + actor = CrossEngineGPUActor(None, None, None, "namespace", 1) + actor.transfer = SimpleNamespace(send_lock=asyncio.Lock(), release_direct=Mock()) + state = actor.rooms[123] = {"completed": asyncio.Event(), "chunks": []} + waiter = asyncio.create_task(actor.wait_done(123)) + await asyncio.sleep(0) + await actor.abort(123) + with pytest.raises(RuntimeError, match="cancelled"): + await waiter + assert not actor.rooms and state["aborted"] From 5c55e954aef98b0e4867b759ca38f54afe599958 Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 02:07:06 +0800 Subject: [PATCH 5/8] test: cover cross-engine PD after merged backend updates --- xinference/core/replica_config.py | 5 ++- xinference/core/tests/test_pd_launch.py | 35 ++++++++++++++++++- .../model/llm/sglang/tests/test_xavier_pd.py | 16 ++++++--- .../model/llm/sglang/xavier/directory.py | 11 +++++- .../llm/vllm/xavier/test/test_cross_engine.py | 5 +-- .../llm/xavier/tests/test_pd_cross_engine.py | 15 ++++++-- .../xavier/tests/test_pd_cross_engine_gpu.py | 9 +++-- 7 files changed, 83 insertions(+), 13 deletions(-) diff --git a/xinference/core/replica_config.py b/xinference/core/replica_config.py index 93557a3c32..e1d45001f9 100644 --- a/xinference/core/replica_config.py +++ b/xinference/core/replica_config.py @@ -190,7 +190,10 @@ def validate_pd_replica_configs( engines = { (cfg.model_engine or model_engine or "").lower() for cfg in configs or [] } - if not engines <= {"vllm", "sglang", "mlx"} or (model_type or "LLM").lower() != "llm": + if ( + not engines <= {"vllm", "sglang", "mlx"} + or (model_type or "LLM").lower() != "llm" + ): raise ValueError( "PD separation requires model_type=LLM and model_engine=vLLM, SGLang or MLX" ) diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index f70ecd369b..60cc94bd26 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -749,11 +749,13 @@ async def test_failed_cache_actor_cleanup_emits_warning(launch_runtime, caplog): @pytest.mark.asyncio @pytest.mark.parametrize("engines", [("vLLM", "SGLang"), ("SGLang", "vLLM")]) +@pytest.mark.parametrize("default_engine", ["vLLM", "SGLang", "MLX"]) async def test_cross_engine_pd_launches_one_adapter_per_replica( - launch_runtime, engines + launch_runtime, engines, default_engine ): supervisor, workers, actors, _ = launch_runtime kwargs = launch_kwargs() + kwargs["model_engine"] = default_engine for index, (cfg, engine) in enumerate(zip(kwargs["replica_config"], engines)): cfg.model_engine = engine cfg.engine_config = {"engine_option": index} @@ -771,6 +773,37 @@ async def test_cross_engine_pd_launches_one_adapter_per_replica( worker.start_transfer_for_vllm.assert_not_awaited() +@pytest.mark.asyncio +@pytest.mark.parametrize("engines", [("MLX", "vLLM"), ("SGLang", "MLX")]) +async def test_cross_engine_mlx_is_rejected_before_allocating_actors( + launch_runtime, engines +): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + for cfg, engine in zip(kwargs["replica_config"], engines): + cfg.model_engine = engine + with pytest.raises(ValueError, match="vLLM and SGLang replicas"): + await supervisor.launch_builtin_model(**kwargs) + assert not actors + for worker in workers: + worker.launch_builtin_model.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_per_replica_engine_override_preserves_homogeneous_mlx_pd(launch_runtime): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + kwargs["model_format"] = "mlx" + for cfg in kwargs["replica_config"]: + cfg.model_engine = "MLX" + await supervisor.launch_builtin_model(**kwargs) + assert set(actors) == {"XavierBytesCacheActor", "PDModelActor"} + for worker in workers: + launch = worker.launch_builtin_model.call_args.kwargs + assert launch["model_engine"] == "MLX" + assert "rank" not in launch["_xavier_cache_config"] + + @pytest.mark.asyncio async def test_cross_engine_rejects_native_transport_before_launch(launch_runtime): supervisor, workers, actors, _ = launch_runtime diff --git a/xinference/model/llm/sglang/tests/test_xavier_pd.py b/xinference/model/llm/sglang/tests/test_xavier_pd.py index 59afd5d266..043f973280 100644 --- a/xinference/model/llm/sglang/tests/test_xavier_pd.py +++ b/xinference/model/llm/sglang/tests/test_xavier_pd.py @@ -372,23 +372,31 @@ async def prepare(h, prompt, room=1): @pytest.mark.asyncio -async def test_repeated_prompt_reuses_fingerprint_and_validates_every_room(): +@pytest.mark.parametrize("heterogeneous", [False, True]) +async def test_repeated_prompt_reuses_fingerprint_and_validates_every_room( + heterogeneous, +): h = fingerprint_handoff() + if heterogeneous: + h.config.update(heterogeneous=True, rank=1) for room in range(1, 4): await prepare(h, "prompt", room) h.tokenizer.encode.assert_called_once_with("prompt") assert h._directory_actor.prepare.await_count == 3 expected = hashlib.sha256(json.dumps(list(b"prompt")).encode()).hexdigest() - assert h._directory_actor.prepare.call_args.args == ( + arguments = ( 3, "namespace", expected, "decode", - None, + 1 if heterogeneous else None, 600, ) + if heterogeneous: + arguments += (len(b"prompt"),) + assert h._directory_actor.prepare.call_args.args == arguments assert list(h._prompt_hashes) == [hashlib.sha256(b"prompt").digest()] - assert list(h._prompt_hashes.values()) == [expected] + assert list(h._prompt_hashes.values()) == [(expected, len(b"prompt"))] @pytest.mark.asyncio diff --git a/xinference/model/llm/sglang/xavier/directory.py b/xinference/model/llm/sglang/xavier/directory.py index b16c811264..d3d21366c3 100644 --- a/xinference/model/llm/sglang/xavier/directory.py +++ b/xinference/model/llm/sglang/xavier/directory.py @@ -62,7 +62,16 @@ def _drop(self, room): state["source_ready"].set() state["complete_ready"].set() - def prepare(self, room, namespace, prompt_hash, role, rank=None, timeout=None, prompt_tokens=None): + def prepare( + self, + room, + namespace, + prompt_hash, + role, + rank=None, + timeout=None, + prompt_tokens=None, + ): self._expire() if namespace != self.namespace or role not in ("prefill", "decode"): raise ValueError("Unregistered SGLang Xavier PD namespace or role") diff --git a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py index cfefd88f70..6fb3666ff7 100644 --- a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py +++ b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py @@ -72,7 +72,7 @@ async def test_cross_prepare_checks_actual_token_ids_once(cross_module): ) instance._prepared = set() instance.contract = SimpleNamespace(fingerprint="a" * 64) - instance._xavier_config = dict(role="decode") + instance._xavier_config = dict(role="decode", rank=1) directory = AsyncMock() instance.directory = AsyncMock(return_value=directory) request = SimpleNamespace( @@ -84,7 +84,8 @@ async def test_cross_prepare_checks_actual_token_ids_once(cross_module): assert await instance._prepare(request) == 9 directory.prepare.assert_awaited_once() args = directory.prepare.call_args.args - assert args == (9, "a" * 64, cross_module.prompt_digest([1, 2, 3]), "decode", 3) + assert args == (9, "a" * 64, cross_module.prompt_digest([1, 2, 3]), "decode") + assert directory.prepare.call_args.kwargs == dict(rank=1, prompt_tokens=3) @pytest.mark.parametrize( diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py index 0540a66c7d..4c429924bb 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -80,8 +80,19 @@ def test_first_token_metadata_and_room_marker(): def test_directory_proves_imported_tokens_and_single_completion(kv_contract): directory = XavierPDDirectory() directory.configure(kv_contract.fingerprint) - for role in ("prefill", "decode"): - directory.prepare(1, kv_contract.fingerprint, prompt_digest([1, 2]), role, prompt_tokens=2) + for rank, role in enumerate(("prefill", "decode")): + directory.prepare( + 1, + kv_contract.fingerprint, + prompt_digest([1, 2]), + role, + rank=rank, + timeout=45, + prompt_tokens=2, + ) + assert directory.rooms[1]["ranks"] == {"prefill": 0, "decode": 1} + assert directory.rooms[1]["timeout"] == 45 + assert directory.request_info(1) == dict(prompt_tokens=2) directory.complete(1, 4096, 1) directory.complete(1, 4096, 1) stats = directory.get_stats() diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py index 6cd7a2fd56..174da39b1e 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py @@ -85,9 +85,14 @@ async def test_gpu_pages_survive_cross_engine_layouts_bitwise(kv_contract, produ ) actors.append(actor) room, source, destination = 123, [5, 1, 3], [2, 6, 4] - for role in ("prefill", "decode"): + for rank, role in enumerate(("prefill", "decode"), 1): await directory.prepare( - room, contract.fingerprint, prompt_digest(list(range(129))), role, 129 + room, + contract.fingerprint, + prompt_digest(list(range(129))), + role, + rank=rank, + prompt_tokens=129, ) if producer == "vllm": await actors[0].register_prefill(room, "request", source, 42, 129) From 15bf602216fea46ca6e348326cf6dd84d97d334e Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 02:15:01 +0800 Subject: [PATCH 6/8] ci: run cross-engine Xavier adapter regressions --- .github/workflows/python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 3d9fed2c1e..0196b0c224 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -395,6 +395,7 @@ jobs: xinference/model/llm/vllm/xavier/test/test_v1_engine.py \ xinference/model/llm/vllm/xavier/test/test_gpu_transfer.py \ xinference/model/llm/vllm/xavier/test/test_kv_schema.py \ + xinference/model/llm/vllm/xavier/test/test_cross_engine.py \ xinference/model/llm/vllm/xavier/test/test_direct_handoff.py \ xinference/model/llm/vllm/xavier/test/test_direct_history.py \ xinference/model/llm/vllm/xavier/test/test_tiered_snapshot.py \ From d0bc1f48be49dc7e9868705e0dfe3fe7a1bc907d Mon Sep 17 00:00:00 2001 From: qinxuye Date: Wed, 7 Oct 2026 03:23:22 +0800 Subject: [PATCH 7/8] fix: handle cross-engine PD boundaries and request failures --- benchmark/benchmark_cross_engine_pd.py | 37 +++- xinference/core/pd_model.py | 45 ++-- xinference/core/replica_config.py | 37 ++++ xinference/core/supervisor.py | 10 +- .../core/tests/test_model_replica_scale.py | 23 +++ xinference/core/tests/test_pd_launch.py | 27 +++ xinference/core/tests/test_pd_model.py | 10 + xinference/model/llm/sglang/xavier/config.py | 61 +++--- .../model/llm/sglang/xavier/directory.py | 11 +- xinference/model/llm/sglang/xavier/gpu.py | 5 +- xinference/model/llm/vllm/core.py | 31 +++ xinference/model/llm/vllm/tests/test_pd.py | 89 ++++++++ .../model/llm/vllm/xavier/cross_engine.py | 129 ++++++++---- .../llm/vllm/xavier/test/test_cross_engine.py | 150 +++++++++++++- .../model/llm/xavier/backends/torch/pd.py | 144 ++++++++++--- xinference/model/llm/xavier/pd_contract.py | 54 ++++- .../llm/xavier/tests/test_pd_cross_engine.py | 193 +++++++++++++++++- .../xavier/tests/test_pd_cross_engine_gpu.py | 51 ++++- 18 files changed, 960 insertions(+), 147 deletions(-) diff --git a/benchmark/benchmark_cross_engine_pd.py b/benchmark/benchmark_cross_engine_pd.py index 1fe244a448..65e6ac2797 100644 --- a/benchmark/benchmark_cross_engine_pd.py +++ b/benchmark/benchmark_cross_engine_pd.py @@ -17,7 +17,7 @@ import xoscar as xo from benchmark_pd import measure, summarize from benchmark_sglang_pd import GPUIsolation, stop_server -from openai import AsyncOpenAI +from openai import APIStatusError, AsyncOpenAI from xoscar.utils import get_next_port from xinference.client import Client @@ -139,6 +139,41 @@ async def requests_and_measure(endpoint, uid, root, worker, mode, args): stream_text="".join(pieces), ) ) + if mode in ("vllm-sglang", "sglang-vllm"): + results["boundary_correctness"] = [] + for tokens in (64, 65, 66, 128, 129, 130, 193): + body = dict( + prompt=" hello" * tokens, + temperature=0, + max_tokens=8, + extra_body=dict(ignore_eos=True), + ) + regular = await api.completions.create(model=uid, **body) + stream = await api.completions.create(model=uid, **body, stream=True) + pieces = [] + async with stream: + async for chunk in stream: + if chunk.choices: + pieces.append(chunk.choices[0].text) + assert regular.usage.prompt_tokens == tokens + assert regular.usage.completion_tokens == 8 + assert regular.choices[0].text == "".join(pieces) + results["boundary_correctness"].append( + dict(prompt_tokens=tokens, text=regular.choices[0].text) + ) + if mode == "sglang-vllm": + try: + await api.completions.create( + model=uid, prompt=" hello", max_tokens=8, temperature=0 + ) + except APIStatusError as error: + assert "at least two prompt tokens" in str(error) + results["rejected_one_token"] = error.status_code + else: + raise AssertionError( + "One-token vLLM decode should be rejected before engine submission" + ) + await api.chat.completions.create(model=uid, **bodies()[0]) for concurrency in args.concurrency: # Repeated prefixes keep native P caching available in every mode. requests_list = [bodies()[i % 4] for i in range(args.requests)] diff --git a/xinference/core/pd_model.py b/xinference/core/pd_model.py index a055da93ac..fb0f05630d 100644 --- a/xinference/core/pd_model.py +++ b/xinference/core/pd_model.py @@ -330,25 +330,32 @@ async def _infer(self, method, inputs, *args, **kwargs): raise ValueError("PD KV handoff currently requires n=1") if self._model_engine == "heterogeneous": config = (args[0] if args else {}) or {} - if any( - config.get(key) is not None and config.get(key) is not False - for key in ("logprobs", "prompt_logprobs") - ) or any( - config.get(key) - for key in ( - "top_logprobs", - "return_logprob", - "top_logprobs_num", - "guided_json", - "guided_regex", - "guided_choice", - "guided_grammar", - "guided_json_object", - "structured_outputs", - "json_schema", - "regex", - "ebnf", - "response_format", + if ( + any( + config.get(key) is not None and config.get(key) is not False + for key in ("logprobs", "prompt_logprobs") + ) + or any( + config.get(key) + for key in ( + "top_logprobs", + "return_logprob", + "top_logprobs_num", + "guided_json", + "guided_regex", + "guided_choice", + "guided_grammar", + "guided_json_object", + "structured_outputs", + "json_schema", + "regex", + "ebnf", + ) + ) + or ( + isinstance(config.get("response_format"), dict) + and config["response_format"].get("type") + in ("json_object", "json_schema") ) ): raise ValueError( diff --git a/xinference/core/replica_config.py b/xinference/core/replica_config.py index e1d45001f9..a8fe95327e 100644 --- a/xinference/core/replica_config.py +++ b/xinference/core/replica_config.py @@ -182,6 +182,43 @@ def validate_pd_replica_configs( model_type: Optional[str], ) -> bool: """Validate a complete P/D topology before allocating any runtime resources.""" + reserved = { + "model_uid", + "model_name", + "model_size_in_billions", + "model_format", + "quantization", + "model_engine", + "model_type", + "model_path", + "n_gpu", + "gpu_idx", + "worker_ip", + "replica", + "replica_uid", + "replica_config", + "role", + "request_limits", + "peft_model_config", + "download_hub", + "enable_virtual_env", + "virtual_env_packages", + "virtual_env_find_links", + "envs", + "xavier_config", + "enable_xavier", + "transfer_backend_type", + "xavier_cache_bytes", + "xavier_gpu_cache_bytes", + } + for cfg in configs or []: + invalid = { + key for key in cfg.engine_config if key in reserved or key.startswith("_") + } + if invalid: + raise ValueError( + f"Reserved replica engine_config keys: {', '.join(sorted(invalid))}" + ) roles = {cfg.role for cfg in configs or []} if not roles.intersection({"prefill", "decode"}): if any(cfg.model_engine or cfg.engine_config for cfg in configs or []): diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index b0bcf40c4c..b0a152f055 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3217,7 +3217,9 @@ async def launch_builtin_model( ) if (sglang_xavier or heterogeneous_pd) and gpu_cache_bytes not in (None, 0): raise ValueError( - "SGLang Xavier GPU PD does not yet support retained GPU history" + "Cross-engine Xavier GPU PD does not yet support retained GPU history" + if heterogeneous_pd + else "SGLang Xavier GPU PD does not yet support retained GPU history" ) if pd_enabled and enable_xavier and gpu_cache_bytes is None: gpu_cache_bytes = 256 * 1024 * 1024 @@ -5025,6 +5027,8 @@ async def add_model_replicas( raise ValueError("The replica count to add must be at least 1") if replica_configs is not None and len(replica_configs) != replica: raise ValueError("replica_configs length must match replica") + if any(cfg.model_engine or cfg.engine_config for cfg in replica_configs or []): + raise ValueError("Per-replica engine settings require explicit PD roles") async with self._get_model_replica_lock(model_uid): results: List[Dict[str, Any]] = [] @@ -5081,6 +5085,10 @@ async def _add_model_replica( raise ValueError( "PD topology cannot be resized in place; terminate and relaunch the model" ) + if replica_config and ( + replica_config.model_engine or replica_config.engine_config + ): + raise ValueError("Per-replica engine settings require explicit PD roles") replica_info = self._model_uid_to_replica_info.get(model_uid) if replica_info is None: raise ValueError(f"Model not found in the model list, uid: {model_uid}") diff --git a/xinference/core/tests/test_model_replica_scale.py b/xinference/core/tests/test_model_replica_scale.py index 1f468d0bff..a67b495b82 100644 --- a/xinference/core/tests/test_model_replica_scale.py +++ b/xinference/core/tests/test_model_replica_scale.py @@ -217,6 +217,29 @@ async def test_scale_up_cpu_placement_normalizes_zero_gpu_count(): assert worker.launches[0]["n_gpu"] is None +@pytest.mark.asyncio +@pytest.mark.parametrize( + "field,value", + [("model_engine", "SGLang"), ("engine_config", {"context_length": 8192})], +) +@pytest.mark.parametrize("count", [1, 2]) +async def test_scale_up_rejects_per_replica_engine_settings_before_any_launch( + field, value, count +): + launch_args = {"n_gpu": None, "gpu_idx": None, "n_worker": 1} + worker = _FakeScaleWorker("worker-0:9978", launch_args) + supervisor = _make_supervisor([worker], launch_args) + config = ReplicaConfig(**{field: value}) + with pytest.raises(ValueError, match="explicit PD roles"): + if count == 1: + await supervisor.add_model_replica("demo", config) + else: + await supervisor.add_model_replicas( + "demo", replica=2, replica_configs=[ReplicaConfig(), config] + ) + assert worker.launches == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("enabled", [True, "true"]) async def test_weight_cached_model_rejects_scale_up_before_allocation(enabled): diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index 60cc94bd26..4eeac1bdf4 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -789,6 +789,33 @@ async def test_cross_engine_mlx_is_rejected_before_allocating_actors( worker.launch_builtin_model.assert_not_awaited() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "key", + [ + "model_engine", + "xavier_config", + "n_gpu", + "gpu_idx", + "envs", + "_xavier_cache_config", + "_nixl_config", + "quantization", + ], +) +async def test_pd_reserved_engine_options_fail_before_allocating_actors( + launch_runtime, key +): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + kwargs["replica_config"][0].engine_config = {key: None} + with pytest.raises(ValueError, match="Reserved replica engine_config keys"): + await supervisor.launch_builtin_model(**kwargs) + assert not actors + for worker in workers: + worker.launch_builtin_model.assert_not_awaited() + + @pytest.mark.asyncio async def test_per_replica_engine_override_preserves_homogeneous_mlx_pd(launch_runtime): supervisor, workers, actors, _ = launch_runtime diff --git a/xinference/core/tests/test_pd_model.py b/xinference/core/tests/test_pd_model.py index b7deb65924..f4a6701df1 100644 --- a/xinference/core/tests/test_pd_model.py +++ b/xinference/core/tests/test_pd_model.py @@ -85,6 +85,16 @@ async def test_infer_preserves_decode_config(router, method): prefill.free_model_cache.assert_not_awaited() +@pytest.mark.asyncio +async def test_cross_engine_allows_plain_text_response_format(router): + actor, prefill, decode = router + actor._model_engine = "heterogeneous" + await actor._infer( + "generate", "prompt", {"response_format": {"type": "text"}}, request_id="r" + ) + assert prefill.generate.await_count == decode.generate.await_count == 1 + + @pytest.mark.asyncio @pytest.mark.parametrize( "engine,backend", diff --git a/xinference/model/llm/sglang/xavier/config.py b/xinference/model/llm/sglang/xavier/config.py index fb9f8b8264..3538d0d9b8 100644 --- a/xinference/model/llm/sglang/xavier/config.py +++ b/xinference/model/llm/sglang/xavier/config.py @@ -114,36 +114,6 @@ def configure_xavier(model_path: str, model_config: dict, cache_config: dict) -> "SGLang Xavier requires local model weights and tokenizer assets" ) model_config.setdefault("page_size", 64) - num_kv_heads = config.get("num_key_value_heads") - head_dim = config.get("head_dim") - contract = KVCacheContract( - weights_fingerprint=fingerprint_files( - {file.name: file for file in weights}, - cache_dir=Path(XINFERENCE_CACHE_DIR) / "xavier-fingerprints", - ), - tokenizer_fingerprint=fingerprint_files( - {file.name: file for file in tokenizer} - ), - attention_fingerprint=fingerprint_metadata(config), - position_fingerprint=fingerprint_metadata( - {"config": config, "context_length": model_config.get("context_length")} - ), - num_layers=config["num_hidden_layers"], - num_kv_heads=( - config["num_attention_heads"] if num_kv_heads is None else num_kv_heads - ), - head_dim=( - config["hidden_size"] // config["num_attention_heads"] - if head_dim is None - else head_dim - ), - block_size=model_config["page_size"], - logical_dtype="float16", - ) - if config.get("torch_dtype", config.get("dtype")) in ("bfloat16", "bf16"): - logger.warning( - "SGLang Xavier casts this BF16 checkpoint to FP16; weights and KV use FP16" - ) if cache_config.get("heterogeneous"): from ...xavier.pd_contract import build_pd_contract @@ -154,6 +124,37 @@ def configure_xavier(model_path: str, model_config: dict, cache_config: dict) -> if model_config["page_size"] != 64: raise ValueError("Cross-engine Xavier PD requires 64-token pages") contract = build_pd_contract(model_path, model_config.get("context_length")) + else: + num_kv_heads = config.get("num_key_value_heads") + head_dim = config.get("head_dim") + contract = KVCacheContract( + weights_fingerprint=fingerprint_files( + {file.name: file for file in weights}, + cache_dir=Path(XINFERENCE_CACHE_DIR) / "xavier-fingerprints", + ), + tokenizer_fingerprint=fingerprint_files( + {file.name: file for file in tokenizer} + ), + attention_fingerprint=fingerprint_metadata(config), + position_fingerprint=fingerprint_metadata( + {"config": config, "context_length": model_config.get("context_length")} + ), + num_layers=config["num_hidden_layers"], + num_kv_heads=( + config["num_attention_heads"] if num_kv_heads is None else num_kv_heads + ), + head_dim=( + config["hidden_size"] // config["num_attention_heads"] + if head_dim is None + else head_dim + ), + block_size=model_config["page_size"], + logical_dtype="float16", + ) + if config.get("torch_dtype", config.get("dtype")) in ("bfloat16", "bf16"): + logger.warning( + "SGLang Xavier casts this BF16 checkpoint to FP16; weights and KV use FP16" + ) model_config["dtype"] = "float16" if cache_config.get("role"): if model_config.get("enable_hierarchical_cache") or any( diff --git a/xinference/model/llm/sglang/xavier/directory.py b/xinference/model/llm/sglang/xavier/directory.py index d3d21366c3..fa4d2b580c 100644 --- a/xinference/model/llm/sglang/xavier/directory.py +++ b/xinference/model/llm/sglang/xavier/directory.py @@ -14,6 +14,7 @@ class XavierPDDirectory(xo.StatelessActor): def __init__(self): super().__init__() self.namespace = None + self.contract = None self.rooms = {} self.completed_requests = 0 self.gpu_bytes = 0 @@ -45,10 +46,18 @@ def unregister_peer(self, rank, address=None): self._drop(room) return True - def configure(self, namespace): + def configure(self, namespace, contract=None): if self.namespace is not None and namespace != self.namespace: + if self.contract is not None and contract is not None: + from ...xavier.contract import KVCacheContract + + KVCacheContract.from_dict(self.contract).require_match( + KVCacheContract.from_dict(contract) + ) raise ValueError("SGLang Xavier GPU KV namespaces differ between replicas") self.namespace = namespace + if contract is not None: + self.contract = contract def _expire(self): now = time.monotonic() diff --git a/xinference/model/llm/sglang/xavier/gpu.py b/xinference/model/llm/sglang/xavier/gpu.py index 6d5cab0f32..85d83fc430 100644 --- a/xinference/model/llm/sglang/xavier/gpu.py +++ b/xinference/model/llm/sglang/xavier/gpu.py @@ -427,7 +427,10 @@ async def _start(self, args, contract): directory = await xo.actor_ref( address=self.config["address"], uid=self.config["uid"] ) - await directory.configure(namespace) + if self.config.get("heterogeneous"): + await directory.configure(namespace, contract.to_dict()) + else: + await directory.configure(namespace) from torch.multiprocessing.reductions import reduce_tensor from xoscar.backends.allocate_strategy import ProcessIndex diff --git a/xinference/model/llm/vllm/core.py b/xinference/model/llm/vllm/core.py index 8ffcca5cd6..965561eb2d 100644 --- a/xinference/model/llm/vllm/core.py +++ b/xinference/model/llm/vllm/core.py @@ -2067,6 +2067,29 @@ async def async_generate( if not request_id: request_id = str(uuid.uuid1()) + xavier_config = getattr(self, "_xavier_config", None) + if ( + xavier_config + and xavier_config.get("heterogeneous") + and generate_config + and "_pd_kv_transfer_params" in generate_config + ): + from ..xavier.pd_contract import prepare_pd_request + + if isinstance(prompt_or_token_ids, str): + prompt_or_token_ids = await self._gen_tokens_prompt( + await self._get_tokenizer(lora_request), + prompt_or_token_ids, + cast(dict, sanitized_generate_config), + ) + if isinstance(prompt_or_token_ids, list): + prompt_or_token_ids = {"prompt_token_ids": prompt_or_token_ids} + await prepare_pd_request( + xavier_config, + cast(Dict[str, Any], prompt_or_token_ids)["prompt_token_ids"], + sampling_params.extra_args["kv_transfer_params"], + ) + assert self._engine is not None start_wall_time = time.time() start_perf = time.perf_counter() @@ -2221,6 +2244,14 @@ async def stream_results() -> AsyncGenerator[CompletionChunk, None]: final_output = request_output assert final_output is not None + transfer = getattr(final_output, "kv_transfer_params", None) + if ( + xavier_config + and xavier_config.get("heterogeneous") + and isinstance(transfer, dict) + and transfer.get("xavier_error") + ): + raise RuntimeError(transfer["xavier_error"]) self._log_pd_request_metrics(final_output) completion = self._convert_request_output_to_completion( request_id, model=self.model_uid, request_output=final_output diff --git a/xinference/model/llm/vllm/tests/test_pd.py b/xinference/model/llm/vllm/tests/test_pd.py index 60f5339ec2..17ff2b4090 100644 --- a/xinference/model/llm/vllm/tests/test_pd.py +++ b/xinference/model/llm/vllm/tests/test_pd.py @@ -117,6 +117,95 @@ async def generate(self, prompt, params, request_id, **kwargs): assert not model._active_request_ids +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", ["one-token", "prompt mismatch", "room limit", "producer RPC"] +) +async def test_cross_engine_validation_fails_request_without_submitting_bad_work( + monkeypatch, failure +): + import xoscar as xo + + from ...xavier.contract import KVCacheContract + from .. import core + + class SamplingParams: + def __init__(self, max_tokens=1, **kwargs): + self.max_tokens = max_tokens + self.__dict__.update(kwargs) + self.extra_args = None + + monkeypatch.setitem( + sys.modules, + "vllm.sampling_params", + SimpleNamespace(SamplingParams=SamplingParams), + ) + monkeypatch.setattr(core, "VLLM_VERSION", Version("0.28.0")) + monkeypatch.setattr(core, "VLLM_INSTALLED", False) + directory = AsyncMock() + if failure in ("prompt mismatch", "room limit"): + directory.prepare.side_effect = ValueError(failure) + monkeypatch.setattr(xo, "actor_ref", AsyncMock(return_value=directory)) + contract = KVCacheContract( + weights_fingerprint="a" * 64, + tokenizer_fingerprint="b" * 64, + attention_fingerprint="c" * 64, + position_fingerprint="d" * 64, + num_layers=2, + num_kv_heads=1, + head_dim=4, + block_size=64, + logical_dtype="float16", + ) + model = object.__new__(core.VLLMModel) + model._nixl_config = None + model._xavier_config = dict( + role="prefill" if failure == "producer RPC" else "decode", + heterogeneous=True, + rank=1, + address="supervisor", + uid="directory", + contract=contract.to_dict(), + gpu_cache_bytes=0, + ) + output = {"xavier_error": failure} if failure == "producer RPC" else None + + async def generate(*args, **kwargs): + yield SimpleNamespace(kv_transfer_params=output) + + model._engine = SimpleNamespace(generate=Mock(side_effect=generate)) + model._active_request_ids, model.lora_requests = set(), [] + model.reasoning_parser, model.model_uid = None, "model" + model._get_tokenizer = AsyncMock() + model._gen_tokens_prompt = AsyncMock( + return_value={"prompt_token_ids": [1] if failure == "one-token" else [1, 2]} + ) + params = { + "max_tokens": 1, + "_pd_kv_transfer_params": {"sglang_xavier": {"mode": "gpu", "room": 123}}, + } + with pytest.raises( + (ValueError, RuntimeError), + match="two prompt tokens" if failure == "one-token" else failure, + ): + await core.VLLMModel.async_generate.__wrapped__( + model, "prompt", params, request_id="bad" + ) + if failure != "producer RPC": + model._engine.generate.assert_not_called() + assert not model._active_request_ids + directory.prepare.side_effect = None + output = None + model._gen_tokens_prompt.return_value = {"prompt_token_ids": [1, 2]} + model._convert_request_output_to_completion = lambda *a, **kw: {"choices": []} + assert ( + await core.VLLMModel.async_generate.__wrapped__( + model, "prompt", params, request_id="good" + ) + is not None + ) + + @pytest.mark.parametrize("address", ["0.0.0.0:9997", ":::9997", "[::]:9997"]) def test_nixl_wildcard_uses_local_interface(monkeypatch, address): from .. import pd diff --git a/xinference/model/llm/vllm/xavier/cross_engine.py b/xinference/model/llm/vllm/xavier/cross_engine.py index e3da8a63c9..4782430864 100644 --- a/xinference/model/llm/vllm/xavier/cross_engine.py +++ b/xinference/model/llm/vllm/xavier/cross_engine.py @@ -67,6 +67,7 @@ class CrossEngineMetadata(KVConnectorMetadata): direct_sends: set[str] = field(default_factory=set) direct_store: bool = False loads: list[tuple[str, int, list[int]]] = field(default_factory=list) + errors: dict[str, str] = field(default_factory=dict) class CrossEngineConnector(XavierConnector): @@ -84,6 +85,8 @@ def __init__(self, vllm_config, role, kv_cache_config): self._cross_requests = {} self._cross_allocated = {} self._prepared = set() + self._cross_prepare_errors = {} + self._pending_aborts = {} async def directory(self): if self._directory is None: @@ -112,7 +115,9 @@ async def _ensure_gpu_cache_mapping(self): self._registered_kv_caches, self._num_cache_blocks, self.contract ) directory = await self.directory() - await directory.configure(self.contract.fingerprint) + await directory.configure( + self.contract.fingerprint, self.contract.to_dict() + ) options = gpu_pool_options(self._xavier_config["host"], os.environ) self._pool = await xo.create_actor_pool( options["external_address"], @@ -153,17 +158,12 @@ async def _prepare(self, request): if not isinstance(handoff, dict) or handoff.get("mode") != "gpu": raise ValueError("Missing cross-engine Xavier GPU room") room = handoff["room"] - if request.request_id not in self._prepared: - directory = await self.directory() - await directory.prepare( - room, - self.contract.fingerprint, - prompt_digest(request.prompt_token_ids), - self._xavier_config["role"], - rank=self._xavier_config["rank"], - prompt_tokens=len(request.prompt_token_ids), + if params.get("xavier_prompt_digest") != prompt_digest( + request.prompt_token_ids + ): + raise ValueError( + "Cross-engine Xavier prompt was not prepared with these token IDs" ) - self._prepared.add(request.request_id) return room def get_num_new_matched_tokens(self, request, num_computed_tokens): @@ -172,12 +172,21 @@ def get_num_new_matched_tokens(self, request, num_computed_tokens): params = request.kv_transfer_params or {} if params.get("do_remote_prefill") is False: return 0, False - if len(request.prompt_token_ids) < 2: - raise ValueError( - "Cross-engine vLLM decode requires at least two prompt tokens" - ) - room = self._call(self._prepare(request)) + try: + if len(request.prompt_token_ids) < 2: + raise ValueError( + "Cross-engine vLLM decode requires at least two prompt tokens" + ) + room = self._call(self._prepare(request)) + except Exception as error: + logger.warning("Cross-engine request preparation failed: %s", error) + self._cross_prepare_errors[request.request_id] = str(error) + room = 0 tokens = max(len(request.prompt_token_ids) - 1 - num_computed_tokens, 0) + if request.request_id in self._cross_prepare_errors: + # Even a one-token request reaching this internal hook must fail + # through the load callback, rather than kill the scheduler. + tokens = max(len(request.prompt_token_ids) - num_computed_tokens, 0) if tokens: self._cross_requests[request.request_id] = room return tokens, bool(tokens) @@ -199,6 +208,8 @@ def build_connector_meta(self, scheduler_output): self._direct_sends = set() for request_id, (room, destinations) in self._cross_allocated.items(): meta.loads.append((request_id, room, destinations)) + if request_id in self._cross_prepare_errors: + meta.errors[request_id] = self._cross_prepare_errors.pop(request_id) self._cross_allocated.clear() return meta @@ -210,10 +221,19 @@ def start_load_kv(self, forward_context, **kwargs): async def submit( room=room, destinations=destinations, request_id=request_id ): - actor = await self._ensure_gpu_cache_mapping() - task = asyncio.create_task( - self._receive_pages(actor, room, destinations) - ) + async def receive(): + try: + if request_id in meta.errors: + raise ValueError(meta.errors[request_id]) + actor = await self._ensure_gpu_cache_mapping() + return await self._receive_pages(actor, room, destinations) + except Exception: + logger.warning( + "Cross-engine KV load submission failed", exc_info=True + ) + return set(destinations) + + task = asyncio.create_task(receive()) self._gpu_load_jobs[task] = [request_id] self._call(submit()) @@ -240,6 +260,7 @@ def wait_for_save(self): def request_finished(self, request, block_ids): self._cross_requests.pop(request.request_id, None) + self._cross_prepare_errors.pop(request.request_id, None) if self._is_producer and (request.kv_transfer_params or {}).get( "do_remote_decode" ): @@ -247,23 +268,39 @@ def request_finished(self, request, block_ids): if request.status == RequestStatus.FINISHED_ABORTED: return False, None - room = self._call(self._prepare(request)) - count = math.ceil(len(request.prompt_token_ids) / self._block_size) - pages = block_ids[:count] - if len(pages) != count or not request.output_token_ids: - raise RuntimeError("Incomplete cross-engine vLLM prefill") - - async def register(): - actor = await self._get_transfer_ref() - await actor.register_prefill( - room, - request.request_id, - pages, - request.output_token_ids[0], - len(request.prompt_token_ids), + room, actor = None, None + try: + room = self._call(self._prepare(request)) + count = math.ceil(len(request.prompt_token_ids) / self._block_size) + pages = block_ids[:count] + if len(pages) != count or not request.output_token_ids: + raise RuntimeError("Incomplete cross-engine vLLM prefill") + actor = self._call(self._get_transfer_ref()) + self._call( + actor.register_prefill( + room, + request.request_id, + pages, + request.output_token_ids[0], + len(request.prompt_token_ids), + ) ) - - self._call(register()) + except Exception as error: + logger.warning("Cross-engine prefill handoff failed", exc_info=True) + hold = False + if actor is not None: + try: + self._call(actor.abort(room)) + except Exception: + # An uncertain registration can still read these slots. + # Keep them pinned until the transfer actor reports done. + logger.warning( + "Cross-engine prefill abort failed", exc_info=True + ) + hold = True + self._direct_sends.add(request.request_id) + self._pending_aborts[request.request_id] = room + return hold, {"xavier_error": str(error)} self._direct_sends.add(request.request_id) self._prepared.discard(request.request_id) return True, dict( @@ -273,6 +310,26 @@ async def register(): self._prepared.discard(request.request_id) return False, None + def get_finished(self, finished_req_ids): + drained = set() + for request_id, room in list(self._pending_aborts.items()): + try: + actor = self._call(self._get_transfer_ref()) + self._call(actor.abort(room)) + except Exception: + logger.warning("Cross-engine abort retry failed", exc_info=True) + else: + drained.add(request_id) + self._direct_sends.discard(request_id) + self._pending_aborts.pop(request_id) + try: + sent, received = super().get_finished(finished_req_ids) + return drained | sent, received + except Exception: + # Preserve pinned requests for a retry if the actor RPC is unavailable. + logger.warning("Cross-engine completion polling failed", exc_info=True) + return drained, set() + def request_finished_all_groups(self, request, block_ids): return self.request_finished(request, list(block_ids[0])) diff --git a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py index 6fb3666ff7..92dac29124 100644 --- a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py +++ b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py @@ -2,6 +2,7 @@ # Licensed under the Apache License, Version 2.0. import asyncio import importlib.util +import json import sys from pathlib import Path from types import SimpleNamespace @@ -36,6 +37,8 @@ def decode(cross_module, tokens): instance._direct_sends = set() instance._direct_handoff = True instance._prepared = set() + instance._cross_prepare_errors = {} + instance._pending_aborts = {} instance._call = lambda coro: coro instance._prepare = lambda request: 123 request = SimpleNamespace( @@ -49,20 +52,20 @@ def decode(cross_module, tokens): def test_vllm_decode_reuses_all_but_final_prompt_token(cross_module): instance, request = decode(cross_module, list(range(193))) assert instance.get_num_new_matched_tokens(request, 0) == (192, True) - blocks = SimpleNamespace(get_block_ids=lambda: ([9, 5, 7, 8],)) + blocks = SimpleNamespace(get_block_ids=lambda: ([9, 5, 7],)) instance.update_state_after_alloc(request, blocks, 192) meta = instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) - assert meta.loads == [("r", 123, [9, 5, 7, 8])] + assert meta.loads == [("r", 123, [9, 5, 7])] assert request.kv_transfer_params["do_remote_prefill"] is False assert instance.get_num_new_matched_tokens(request, 0) == (0, False) # Preemption subsequently recomputes locally; the consumed room is single-use. assert instance._cross_allocated == {} -def test_vllm_decode_rejects_zero_reuse_prompt(cross_module): +def test_vllm_decode_reports_zero_reuse_prompt_as_load_failure(cross_module): instance, request = decode(cross_module, [1]) - with pytest.raises(ValueError, match="two prompt tokens"): - instance.get_num_new_matched_tokens(request, 0) + assert instance.get_num_new_matched_tokens(request, 0) == (1, True) + assert "two prompt tokens" in instance._cross_prepare_errors["r"] @pytest.mark.asyncio @@ -78,14 +81,17 @@ async def test_cross_prepare_checks_actual_token_ids_once(cross_module): request = SimpleNamespace( request_id="r", prompt_token_ids=[1, 2, 3], - kv_transfer_params={"sglang_xavier": dict(mode="gpu", room=9)}, + kv_transfer_params={ + "sglang_xavier": dict(mode="gpu", room=9), + "xavier_prompt_digest": cross_module.prompt_digest([1, 2, 3]), + }, ) assert await instance._prepare(request) == 9 assert await instance._prepare(request) == 9 - directory.prepare.assert_awaited_once() - args = directory.prepare.call_args.args - assert args == (9, "a" * 64, cross_module.prompt_digest([1, 2, 3]), "decode") - assert directory.prepare.call_args.kwargs == dict(rank=1, prompt_tokens=3) + directory.prepare.assert_not_awaited() + request.prompt_token_ids.append(4) + with pytest.raises(ValueError, match="these token IDs"): + await instance._prepare(request) @pytest.mark.parametrize( @@ -132,3 +138,127 @@ async def test_failed_load_reports_all_blocks_without_crashing_vllm( await task assert instance.get_finished(set()) == (set(), {"next"}) assert instance.get_block_ids_with_load_errors() == set() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_stage", ["prepare", "mapping"]) +async def test_hook_failure_finishes_only_its_request(cross_module, failed_stage): + instance, request = decode(cross_module, list(range(193))) + instance._loop = object() + instance._gpu_load_jobs = {} + instance._invalid_block_ids = set() + if failed_stage == "prepare": + + def fail(request): + raise ValueError("prompt mismatch") + + instance._prepare = fail + else: + instance._ensure_gpu_cache_mapping = AsyncMock( + side_effect=OSError("actor unavailable") + ) + matched, pending = instance.get_num_new_matched_tokens(request, 0) + assert pending + destinations = [9, 5, 7, 8][: (matched + 63) // 64] + instance.update_state_after_alloc( + request, SimpleNamespace(get_block_ids=lambda: (destinations,)), matched + ) + meta = instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) + instance._get_connector_metadata = lambda: meta + coroutines = [] + instance._call = coroutines.append + instance.start_load_kv(None) + for coroutine in coroutines: + await coroutine + await asyncio.gather(*instance._gpu_load_jobs) + instance._call = lambda coro: coro.close() + assert instance.get_finished(set()) == (set(), {"r"}) + assert instance.get_block_ids_with_load_errors() == set(destinations) + + +def test_producer_registration_failure_is_drained_without_engine_exception( + cross_module, monkeypatch +): + monkeypatch.setitem( + sys.modules, + "vllm.v1.request", + SimpleNamespace(RequestStatus=SimpleNamespace(FINISHED_ABORTED="aborted")), + ) + instance, request = decode(cross_module, [1, 2]) + instance._is_producer, instance._is_consumer = True, False + instance._block_size = 64 + request.status, request.output_token_ids = "finished", [42] + request.kv_transfer_params["do_remote_decode"] = True + actor = AsyncMock() + actor.register_prefill.side_effect = OSError("reply lost") + actor.abort.side_effect = OSError("temporary disconnect") + instance._get_transfer_ref = AsyncMock(return_value=actor) + instance._prepare = AsyncMock(return_value=123) + instance._call = asyncio.run + hold, metadata = instance.request_finished(request, [9]) + assert hold and metadata["xavier_error"] == "reply lost" + assert instance._pending_aborts == {"r": 123} + instance._direct_handoff = False + instance._gpu_load_jobs = {} + actor.abort.side_effect = None + assert instance.get_finished(set()) == ({"r"}, set()) + assert not instance._pending_aborts and not instance._direct_sends + + +def test_completion_rpc_failure_retains_pinned_request_for_retry(cross_module): + instance, _ = decode(cross_module, [1, 2]) + instance._is_producer, instance._is_consumer = True, False + instance._direct_sends = {"r"} + instance._gpu_load_jobs = {} + instance._transfer_ref = AsyncMock() + instance._transfer_ref.poll_direct_gpu_v1.side_effect = OSError("RPC failed") + instance._call = asyncio.run + assert instance.get_finished(set()) == (set(), set()) + assert instance._direct_sends == {"r"} + instance._transfer_ref.poll_direct_gpu_v1.side_effect = None + instance._transfer_ref.poll_direct_gpu_v1.return_value = {"r"} + assert instance.get_finished(set()) == ({"r"}, set()) + + +@pytest.mark.parametrize("context", [None, 4096, 8192]) +def test_engine_adapters_share_contract_and_cache_weight_hashes( + cross_module, tmp_path, monkeypatch, context +): + from xinference.model.llm.sglang.xavier import config as sglang_config + from xinference.model.llm.xavier import pd_contract + + config = dict( + model_type="qwen2", + num_hidden_layers=2, + num_attention_heads=2, + hidden_size=8, + head_dim=None, + num_key_value_heads=None, + layer_types=None, + max_position_embeddings=4096, + ) + (tmp_path / "config.json").write_text(json.dumps(config)) + (tmp_path / "model.safetensors").write_bytes(b"weights") + (tmp_path / "tokenizer.json").write_bytes(b"tokenizer") + monkeypatch.setattr(pd_contract, "XINFERENCE_CACHE_DIR", str(tmp_path / "cache")) + from unittest.mock import Mock + + old_open = Path.open + reads = [] + + def opened(path, *args, **kwargs): + if path.name == "model.safetensors": + reads.append(path) + return old_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", opened) + duplicate = Mock(side_effect=AssertionError("unused homogeneous contract")) + monkeypatch.setattr(sglang_config, "fingerprint_files", duplicate) + vcache, scache = dict(role="decode"), dict(role="decode", heterogeneous=True) + cross_module.configure_cross_engine( + str(tmp_path), dict(max_model_len=context), vcache + ) + sglang_config.configure_xavier(str(tmp_path), dict(context_length=context), scache) + assert vcache["contract"] == scache["contract"] + assert len(reads) == 1 + duplicate.assert_not_called() diff --git a/xinference/model/llm/xavier/backends/torch/pd.py b/xinference/model/llm/xavier/backends/torch/pd.py index b9b5d0fc74..9e336303bb 100644 --- a/xinference/model/llm/xavier/backends/torch/pd.py +++ b/xinference/model/llm/xavier/backends/torch/pd.py @@ -3,6 +3,8 @@ """Canonical per-layer K/V pages, using Xavier's persistent GPU NIXL slabs.""" import asyncio +import logging +import math import os import re import struct @@ -14,10 +16,13 @@ import xoscar as xo from ....sglang.gc_lifecycle import InitializationGCFreeze +from ....sglang.xavier.settings import transfer_timeout from .direct_handoff import DirectGPUTransfer from .gpu_transfer import finish_before_cancel from .snapshot import block_major_view +logger = logging.getLogger(__name__) + def sglang_first_token_payload(sizes, room, token, prompt_tokens): # SGLang >=0.5.21 MetadataBuffers: output_ids, cached_tokens, logprobs, @@ -142,6 +147,8 @@ async def open(self, room): total=None, sent=0, completed=asyncio.Event(), + changed=asyncio.Event(), + deadline=time.monotonic() + transfer_timeout(), ) try: await self.directory.publish_source( @@ -158,7 +165,9 @@ async def register_prefill( self.init(room, len(pages), 0) state = self.rooms[room] ticket = f"{room}:0" - self.transfer.register_direct(ticket, request_id, pages) + self.transfer.register_direct( + ticket, request_id, pages, lease_timeout=transfer_timeout() + ) state["sent"] = len(pages) state["chunks"].append( dict( @@ -168,13 +177,27 @@ async def register_prefill( aux=dict(first_token=first_token, prompt_tokens=prompt_tokens), ) ) + state["changed"].set() async def poll_direct_gpu_v1(self): finished = self.transfer.poll_direct() for room, state in list(self.rooms.items()): - if state["completed"].is_set(): - await self.directory.release(room, "prefill") + expired = any( + chunk["ticket"] not in self.transfer.direct_requests + and chunk["ticket"] not in state["released"] + for chunk in state["chunks"] + ) + if expired: + await self.abort(room) + finished.update(self.transfer.poll_direct()) + if expired or state["completed"].is_set(): self.rooms.pop(room, None) + try: + await self.directory.release(room, "prefill") + except Exception: + logger.warning( + "Cross-engine room release failed: %s", room, exc_info=True + ) return finished def init(self, room, count, aux_index): @@ -188,6 +211,7 @@ async def add_chunk(self, room, pages, aux_payload: list[bytes] | None = None): if state.get("aborted"): raise RuntimeError("Cross-engine producer was cancelled") state["sent"] += len(pages) + state["deadline"] = time.monotonic() + transfer_timeout() final = state["sent"] == state["total"] if state["sent"] > state["total"]: raise ValueError("SGLang Xavier sent too many KV pages") @@ -203,9 +227,12 @@ async def add_chunk(self, room, pages, aux_payload: list[bytes] | None = None): aux = aux_payload if aux is None: raise ValueError("Missing SGLang Xavier first-token metadata") - self.transfer.register_direct(ticket, ticket, pages) + self.transfer.register_direct( + ticket, ticket, pages, lease_timeout=transfer_timeout() + ) # Scheduler-owned source slots remain pinned until every chunk is read. state["chunks"].append(dict(ticket=ticket, pages=pages, final=final, aux=aux)) + state["changed"].set() def chunk(self, room, index): state = self.rooms.get(room) @@ -213,16 +240,42 @@ def chunk(self, room, index): raise RuntimeError("SGLang Xavier producer was cancelled") return state["chunks"][index] if index < len(state["chunks"]) else None + async def wait_chunk(self, room, index): + state = self.rooms[room] + state["changed"].clear() + chunk = self.chunk(room, index) + while chunk is None: + deadline = state["deadline"] + try: + await asyncio.wait_for( + state["changed"].wait(), timeout=max(0, deadline - time.monotonic()) + ) + except asyncio.TimeoutError: + if state["deadline"] > time.monotonic(): + continue + raise TimeoutError("Cross-engine producer KV transfer timed out") + state["changed"].clear() + chunk = self.chunk(room, index) + return chunk + async def send_direct_gpu_v1(self, ticket, reads, remote_ref, slab_bytes): - return await self.transfer.run( + result = await self.transfer.run( self.transfer.send_direct, ticket, reads, remote_ref, slab_bytes ) + state = self.transfer.direct_requests.get(ticket) + if state is not None: + state.deadline = time.monotonic() + transfer_timeout() + room = self.rooms.get(int(ticket.split(":", 1)[0])) + if room is not None: + room["deadline"] = time.monotonic() + transfer_timeout() + return result def release_chunk(self, ticket): self.transfer.release_direct(ticket) room = int(ticket.split(":", 1)[0]) state = self.rooms.get(room) if state: + state["deadline"] = time.monotonic() + transfer_timeout() state["released"].add(ticket) if state["sent"] == state["total"] and len(state["released"]) == len( state["chunks"] @@ -244,7 +297,18 @@ def done(self, room): async def wait_done(self, room): state = self.rooms[room] - await state["completed"].wait() + while not state["completed"].is_set(): + deadline = state["deadline"] + try: + await asyncio.wait_for( + state["completed"].wait(), + timeout=max(0, deadline - time.monotonic()), + ) + except asyncio.TimeoutError: + if state["deadline"] > time.monotonic(): + continue + await self.abort(room) + raise TimeoutError("Cross-engine producer KV transfer timed out") if state.get("aborted"): raise RuntimeError("SGLang Xavier producer was cancelled") return self.done(room) @@ -259,14 +323,20 @@ def get_stats(self): ) async def _receive(self, room, destinations, aux_index): - deadline = time.monotonic() + 120 - source = None - while source is None: - source = await self.directory.source(room) - if time.monotonic() > deadline: - raise TimeoutError("SGLang Xavier producer bootstrap timed out") - if source is None: - await asyncio.sleep(0.005) + info = await self.directory.request_info(room) + prompt_tokens = info["prompt_tokens"] + source_pages = math.ceil(prompt_tokens / self.contract.block_size) + receive_pages = ( + source_pages + if self.args.aux_item_lens + else math.ceil((prompt_tokens - 1) / self.contract.block_size) + ) + if len(destinations) < receive_pages or len(destinations) > source_pages: + raise ValueError("Cross-engine KV destination page counts differ") + destinations = destinations[:receive_pages] + source = await asyncio.wait_for( + self.directory.wait_source(room), timeout=transfer_timeout() + ) rank = source["rank"] self._world_addresses[rank] = source["address"] sender = await xo.actor_ref( @@ -274,41 +344,48 @@ async def _receive(self, room, destinations, aux_index): ) offset, index, nbytes = 0, 0, 0 while True: - chunk = await sender.chunk(room, index) - if chunk is None: - if time.monotonic() > deadline: - raise TimeoutError("SGLang Xavier producer KV transfer timed out") - await asyncio.sleep(0.001) - continue + chunk = await asyncio.wait_for( + sender.wait_chunk(room, index), timeout=transfer_timeout() + ) pages = chunk["pages"] targets = destinations[offset : offset + len(pages)] - if len(targets) != len(pages): + if offset + len(pages) > source_pages: raise ValueError( "SGLang Xavier source and destination page counts differ" ) - mapping = dict(zip(pages, targets)) - if len(mapping) != len(pages): + mapping = dict(zip(pages[: len(targets)], targets)) + if len(mapping) != len(targets) or len(set(pages)) != len(pages): raise ValueError("Duplicate SGLang Xavier source pages") - invalid = await self.transfer.run( - self.transfer.load_direct, - [{rank: {name: mapping for name in self.caches}}], - [chunk["ticket"]], - ) + if mapping: + invalid = await asyncio.wait_for( + self.transfer.run( + self.transfer.load_direct, + [{rank: {name: mapping for name in self.caches}}], + [chunk["ticket"]], + ), + timeout=transfer_timeout(), + ) + else: + # A 64k+1 prompt has one source-only final page: vLLM + # recomputes its final token after the external load. + await asyncio.wait_for( + sender.release_chunk(chunk["ticket"]), timeout=transfer_timeout() + ) + invalid = set() if invalid: raise RuntimeError("SGLang Xavier GPU handoff is unavailable") offset += len(pages) - nbytes += len(pages) * sum( + nbytes += len(targets) * sum( cache[0].numel() * cache.element_size() for cache in self.caches.values() ) index += 1 if chunk["final"]: + if offset != source_pages: + raise ValueError("Cross-engine KV source page counts differ") if not self.args.aux_item_lens: - if offset != len(destinations): - raise ValueError("Cross-engine KV page counts differ") - stats = await self.directory.request_info(room) await self.directory.complete( - room, nbytes, max(stats["prompt_tokens"] - 1, 0) + room, nbytes, max(prompt_tokens - 1, 0) ) await self.directory.release(room, "decode") return set() @@ -357,6 +434,7 @@ async def abort(self, room): for chunk in state["chunks"]: self.transfer.release_direct(chunk["ticket"]) state["completed"].set() + state["changed"].set() # Native bootstrap aborts need not call sender.clear(). Waiters # already hold the state and will observe its aborted marker. self.rooms.pop(room, None) diff --git a/xinference/model/llm/xavier/pd_contract.py b/xinference/model/llm/xavier/pd_contract.py index faef3fa309..6116094ef6 100644 --- a/xinference/model/llm/xavier/pd_contract.py +++ b/xinference/model/llm/xavier/pd_contract.py @@ -6,6 +6,7 @@ import json from pathlib import Path +from ....constants import XINFERENCE_CACHE_DIR from .contract import KVCacheContract, fingerprint_files, fingerprint_metadata @@ -24,7 +25,7 @@ def build_pd_contract( or config.get("vision_config") or config.get("quantization_config") or config.get("auto_map") - or any(kind != "full_attention" for kind in config.get("layer_types", [])) + or any(kind != "full_attention" for kind in (config.get("layer_types") or [])) ): raise ValueError( "Cross-engine Xavier PD requires full-attention Qwen2, Qwen3 or Llama text weights" @@ -54,7 +55,10 @@ def build_pd_contract( "Cross-engine Xavier PD requires local weights and tokenizer assets" ) return KVCacheContract( - weights_fingerprint=fingerprint_files({file.name: file for file in weights}), + weights_fingerprint=fingerprint_files( + {file.name: file for file in weights}, + cache_dir=Path(XINFERENCE_CACHE_DIR) / "xavier-fingerprints", + ), tokenizer_fingerprint=fingerprint_files( {file.name: file for file in tokenizer} ), @@ -67,10 +71,50 @@ def build_pd_contract( } ), num_layers=config["num_hidden_layers"], - num_kv_heads=config.get("num_key_value_heads", config["num_attention_heads"]), - head_dim=config.get( - "head_dim", config["hidden_size"] // config["num_attention_heads"] + num_kv_heads=( + config["num_attention_heads"] + if config.get("num_key_value_heads") is None + else config["num_key_value_heads"] + ), + head_dim=( + config["hidden_size"] // config["num_attention_heads"] + if config.get("head_dim") is None + else config["head_dim"] ), block_size=64, logical_dtype="float16", ) + + +async def prepare_pd_request( + cache_config: dict, tokens: list[int], params: dict +) -> None: + """Reject incompatible rooms in the API process before entering EngineCore.""" + import asyncio + + import xoscar as xo + + from ..sglang.xavier.settings import transfer_timeout + + if cache_config["role"] == "decode" and len(tokens) < 2: + raise ValueError("Cross-engine vLLM decode requires at least two prompt tokens") + handoff = params.get("sglang_xavier") + if not isinstance(handoff, dict) or handoff.get("mode") != "gpu": + raise ValueError("Missing cross-engine Xavier GPU room") + digest = prompt_digest(tokens) + directory = await xo.actor_ref( + address=cache_config["address"], uid=cache_config["uid"] + ) + await asyncio.wait_for( + directory.prepare( + handoff["room"], + KVCacheContract.from_dict(cache_config["contract"]).fingerprint, + digest, + cache_config["role"], + rank=cache_config["rank"], + timeout=transfer_timeout(), + prompt_tokens=len(tokens), + ), + timeout=transfer_timeout(), + ) + params["xavier_prompt_digest"] = digest diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py index 4c429924bb..01a1d5546b 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -19,6 +19,26 @@ from ..pd_contract import build_pd_contract, prompt_digest +def cpu_actor(kv_contract, directory, rank): + actor = CrossEngineGPUActor( + SimpleNamespace(aux_item_lens=[]), + replace(kv_contract, block_size=64), + directory, + "namespace", + rank, + ) + actor.transfer = SimpleNamespace( + register_direct=Mock(), + release_direct=Mock(), + direct_requests={}, + poll_direct=Mock(return_value=set()), + send_lock=asyncio.Lock(), + metrics={}, + ) + actor.chunk_capacity = 1 + return actor + + @pytest.mark.parametrize("layout", ["2NTHD", "N2THD", "BHN2D", "BHN2D_NHD"]) def test_vllm_canonical_views_preserve_engine_storage(kv_contract, layout): contract = replace(kv_contract, num_layers=2) @@ -151,10 +171,181 @@ async def test_aborted_source_cannot_register_late_gpu_chunks(): async def test_bootstrap_abort_clears_room_and_wakes_existing_waiter(): actor = CrossEngineGPUActor(None, None, None, "namespace", 1) actor.transfer = SimpleNamespace(send_lock=asyncio.Lock(), release_direct=Mock()) - state = actor.rooms[123] = {"completed": asyncio.Event(), "chunks": []} + state = actor.rooms[123] = { + "completed": asyncio.Event(), + "changed": asyncio.Event(), + "deadline": asyncio.get_running_loop().time() + 1, + "chunks": [], + } waiter = asyncio.create_task(actor.wait_done(123)) await asyncio.sleep(0) await actor.abort(123) with pytest.raises(RuntimeError, match="cancelled"): await waiter assert not actor.rooms and state["aborted"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("prompt_tokens", [64, 65, 128, 129, 193]) +@pytest.mark.parametrize("split_final", [False, True]) +async def test_vllm_imports_only_allocated_prefix_pages( + kv_contract, monkeypatch, prompt_tokens, split_final +): + directory = AsyncMock() + directory.request_info.return_value = dict(prompt_tokens=prompt_tokens) + directory.wait_source.return_value = dict(address="source", rank=1) + source = cpu_actor(kv_contract, directory, 1) + destination = cpu_actor(kv_contract, directory, 2) + source_count = (prompt_tokens + 63) // 64 + target_count = (prompt_tokens - 1 + 63) // 64 + source.caches = {"K": torch.arange(7 * 64, dtype=torch.float16).reshape(7, 64)} + destination.caches = {"K": torch.zeros(7, 64, dtype=torch.float16)} + pages, targets = [5, 1, 3, 0][:source_count], [2, 6, 4, 0][:target_count] + await source.open(123) + source.init(123, len(pages), 0) + if split_final and len(pages) > 1: + await source.add_chunk(123, pages[:-1]) + await source.add_chunk(123, pages[-1:], [b"metadata"]) + else: + await source.add_chunk(123, pages, [b"metadata"]) + peer = SimpleNamespace( + wait_chunk=source.wait_chunk, + release_chunk=AsyncMock(side_effect=source.release_chunk), + ) + monkeypatch.setattr("xoscar.actor_ref", AsyncMock(return_value=peer)) + + async def load(requests, tickets): + mapping = requests[0][1]["K"] + for original, target in mapping.items(): + destination.caches["K"][target].copy_(source.caches["K"][original]) + source.release_chunk(tickets[0]) + return set() + + async def run(fn, *args): + return await fn(*args) + + destination.transfer.load_direct, destination.transfer.run = load, run + assert await destination.receive(123, targets, 0) == set() + assert torch.equal( + destination.caches["K"][targets], source.caches["K"][pages[:target_count]] + ) + assert source.rooms[123]["completed"].is_set() + assert source.transfer.release_direct.call_count == len(source.rooms[123]["chunks"]) + directory.complete.assert_awaited_once_with( + 123, target_count * 64 * 2, prompt_tokens - 1 + ) + directory.source.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_progress_and_configured_timeout_control_chunk_wait( + kv_contract, monkeypatch +): + monkeypatch.setenv("XINFERENCE_SGLANG_XAVIER_TRANSFER_TIMEOUT", "1000") + actor = cpu_actor(kv_contract, AsyncMock(), 1) + await actor.open(123) + actor.init(123, 2, 0) + waiting = asyncio.create_task(actor.wait_chunk(123, 0)) + await asyncio.sleep(0) + await actor.add_chunk(123, [1]) + assert (await asyncio.wait_for(waiting, 1))["pages"] == [1] + actor.transfer.register_direct.assert_called_once_with( + "123:0", "123:0", [1], lease_timeout=1000 + ) + actor.rooms[123]["deadline"] = 0 + with pytest.raises(TimeoutError, match="transfer timed out"): + await actor.wait_chunk(123, 1) + with pytest.raises(TimeoutError, match="transfer timed out"): + await actor.wait_done(123) + assert not actor.rooms + + +@pytest.mark.asyncio +async def test_wait_done_honors_progress_refreshed_deadline(kv_contract, monkeypatch): + actor = cpu_actor(kv_contract, AsyncMock(), 1) + await actor.open(123) + actor.init(123, 1, 0) + state = actor.rooms[123] + state["deadline"] = 0 + original = asyncio.wait_for + calls = 0 + + async def refresh(awaitable, timeout): + nonlocal calls + calls += 1 + if calls == 1: + awaitable.close() + await actor.add_chunk(123, [1], [b"metadata"]) + raise asyncio.TimeoutError() + actor.release_chunk("123:0") + return await original(awaitable, timeout) + + monkeypatch.setattr(asyncio, "wait_for", refresh) + assert await actor.wait_done(123) + assert calls == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("expired", [False, True]) +async def test_producer_poll_keeps_finished_ids_when_release_rpc_fails( + kv_contract, expired +): + directory = AsyncMock() + directory.release.side_effect = OSError("supervisor unavailable") + actor = cpu_actor(kv_contract, directory, 1) + await actor.register_prefill(123, "request", [1], 42, 40) + if not expired: + actor.transfer.direct_requests["123:0"] = object() + actor.release_chunk("123:0") + actor.transfer.poll_direct.return_value = {"request"} + assert await actor.poll_direct_gpu_v1() == {"request"} + assert not actor.rooms + directory.release.assert_awaited_once_with(123, "prefill") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "tokens,error", + [ + ([1], "two prompt tokens"), + ([1, 2], "prompt mismatch"), + ([1, 2, 3], "room limit"), + ], +) +async def test_request_preparation_errors_stay_outside_engine( + kv_contract, monkeypatch, tokens, error +): + import xoscar as xo + + from ..pd_contract import prepare_pd_request + + directory = AsyncMock() + directory.prepare.side_effect = ValueError(error) + monkeypatch.setattr(xo, "actor_ref", AsyncMock(return_value=directory)) + config = dict( + role="decode", + rank=2, + address="supervisor", + uid="directory", + contract=kv_contract.to_dict(), + ) + params = dict(sglang_xavier=dict(room=123, mode="gpu")) + with pytest.raises(ValueError, match=error): + await prepare_pd_request(config, tokens, params) + assert "xavier_prompt_digest" not in params + if len(tokens) < 2: + directory.prepare.assert_not_awaited() + directory.prepare.side_effect = None + await prepare_pd_request(config, [1, 2], params) + assert params["xavier_prompt_digest"] == prompt_digest([1, 2]) + + +@pytest.mark.parametrize( + "field", ["weights_fingerprint", "tokenizer_fingerprint", "position_fingerprint"] +) +def test_cross_engine_namespace_mismatch_names_contract_field(kv_contract, field): + directory = XavierPDDirectory() + directory.configure(kv_contract.fingerprint, kv_contract.to_dict()) + other = replace(kv_contract, **{field: "f" * 64}) + with pytest.raises(ValueError, match=field): + directory.configure(other.fingerprint, other.to_dict()) diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py index 174da39b1e..1cfeb72f5e 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py @@ -3,6 +3,7 @@ """Opt-in bitwise tests: XINFERENCE_TEST_CROSS_ENGINE_GPU=1 pytest this file.""" import os +import struct from dataclasses import replace from types import SimpleNamespace @@ -23,7 +24,10 @@ @pytest.mark.asyncio @pytest.mark.parametrize("producer", ["vllm", "sglang"]) -async def test_gpu_pages_survive_cross_engine_layouts_bitwise(kv_contract, producer): +@pytest.mark.parametrize("prompt_tokens", [65, 129, 193]) +async def test_gpu_pages_survive_cross_engine_layouts_bitwise( + kv_contract, producer, prompt_tokens +): from torch.multiprocessing.reductions import reduce_tensor from xoscar.backends.allocate_strategy import ProcessIndex @@ -68,10 +72,16 @@ async def test_gpu_pages_survive_cross_engine_layouts_bitwise(kv_contract, produ XavierPDDirectory, address=pool.external_address ) await directory.configure(contract.fingerprint) + metadata_sizes = [64, 64, 64, 64, 512, 512, 64, 128, 1024, 64] for gpu in (0, 1): actor = await xo.create_actor( CrossEngineGPUActor, - SimpleNamespace(gpu_id=gpu, aux_item_lens=[]), + SimpleNamespace( + gpu_id=gpu, + aux_item_lens=( + metadata_sizes if gpu == 1 and producer == "vllm" else [] + ), + ), contract, directory, contract.fingerprint, @@ -84,31 +94,54 @@ async def test_gpu_pages_survive_cross_engine_layouts_bitwise(kv_contract, produ uid=f"{CrossEngineGPUActor.default_uid()}-{gpu + 1}", ) actors.append(actor) - room, source, destination = 123, [5, 1, 3], [2, 6, 4] + source_count = (prompt_tokens + 63) // 64 + target_count = ( + source_count if producer == "vllm" else (prompt_tokens - 1 + 63) // 64 + ) + room, source, destination = ( + 123, + [5, 1, 3, 0][:source_count], + [2, 6, 4, 0][:target_count], + ) for rank, role in enumerate(("prefill", "decode"), 1): await directory.prepare( room, contract.fingerprint, - prompt_digest(list(range(129))), + prompt_digest(list(range(prompt_tokens))), role, rank=rank, - prompt_tokens=129, + prompt_tokens=prompt_tokens, ) if producer == "vllm": - await actors[0].register_prefill(room, "request", source, 42, 129) + await actors[0].register_prefill(room, "request", source, 42, prompt_tokens) else: await actors[0].open(room) await actors[0].init(room, len(source), 0) await actors[0].add_chunk(room, source[:1]) await actors[0].add_chunk(room, source[1:], [b"first-token"]) - assert await actors[1].receive(room, destination, 0) == set() + received = await actors[1].receive(room, destination, 0) + if producer == "vllm": + nbytes, payload = received + assert struct.unpack_from(" Date: Wed, 7 Oct 2026 10:00:09 +0800 Subject: [PATCH 8/8] fix: carry PD cancellation retries across connector roles --- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- .../LC_MESSAGES/user_guide/pd_separation.po | 2 +- doc/source/user_guide/pd_separation.rst | 2 +- xinference/core/replica_config.py | 3 + xinference/core/supervisor.py | 15 ++-- xinference/core/tests/test_pd_launch.py | 32 ++++++++ xinference/core/tests/test_pd_model.py | 2 + xinference/model/llm/vllm/tests/test_pd.py | 14 ++++ .../model/llm/vllm/xavier/cross_engine.py | 30 +++---- .../llm/vllm/xavier/test/test_cross_engine.py | 82 +++++++++++++++---- .../llm/vllm/xavier/test/test_v1_engine.py | 19 +++++ .../model/llm/xavier/backends/torch/pd.py | 24 +++--- .../llm/xavier/tests/test_pd_cross_engine.py | 31 +++++-- .../xavier/tests/test_pd_cross_engine_gpu.py | 13 ++- 21 files changed, 212 insertions(+), 73 deletions(-) diff --git a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po index 587a25c898..ba095b8e8f 100644 --- a/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/de/LC_MESSAGES/user_guide/pd_separation.po @@ -596,7 +596,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po index 0133c7c09f..faa386e1f9 100644 --- a/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/es/LC_MESSAGES/user_guide/pd_separation.po @@ -590,7 +590,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po index c2b6406c7f..826c6b443a 100644 --- a/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/fr/LC_MESSAGES/user_guide/pd_separation.po @@ -598,7 +598,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po index f1c2cbffec..eb0b346bb0 100644 --- a/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/it/LC_MESSAGES/user_guide/pd_separation.po @@ -592,7 +592,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po index a3233ae909..2843d88f82 100644 --- a/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ja/LC_MESSAGES/user_guide/pd_separation.po @@ -533,7 +533,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po index 5ff1039a3e..3a9bbd93d4 100644 --- a/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/ko/LC_MESSAGES/user_guide/pd_separation.po @@ -531,7 +531,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po index 94622a9e6e..550b08794e 100644 --- a/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/pt_BR/LC_MESSAGES/user_guide/pd_separation.po @@ -586,7 +586,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po index c2cae5aac6..1785aa9d79 100644 --- a/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_CN/LC_MESSAGES/user_guide/pd_separation.po @@ -518,7 +518,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po index 86f2012197..7590f205c9 100644 --- a/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po +++ b/doc/source/locale/zh_TW/LC_MESSAGES/user_guide/pd_separation.po @@ -516,7 +516,7 @@ msgstr "" #: source/user_guide/pd_separation.rst msgid "" -"Cross-engine P/D supports no CPU fallback, retained Xavier history, " +"Cross-engine P/D does not support CPU fallback, retained Xavier history, " "logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs" " or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do" " not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native " diff --git a/doc/source/user_guide/pd_separation.rst b/doc/source/user_guide/pd_separation.rst index ce8cf7b193..ad95577a81 100644 --- a/doc/source/user_guide/pd_separation.rst +++ b/doc/source/user_guide/pd_separation.rst @@ -138,7 +138,7 @@ Both workers require identical local weights, tokenizer assets and context limit Streaming and non-streaming requests require ``n=1``. SGLang decode uses the first output token sampled by prefill. vLLM decode imports all but the final prompt token, computes that token and samples the output. Engine kernels can produce small numerical differences. vLLM decode requires at least two prompt tokens. -Cross-engine P/D supports no CPU fallback, retained Xavier history, logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native NIXL comparisons use the same engine on both roles. MLX is unsupported. +Cross-engine P/D does not support CPU fallback, retained Xavier history, logprobs, structured sampling, LoRA, speculative decoding, multimodal inputs or hybrid attention. Omit ``xavier_gpu_cache_bytes`` or set it to ``0``; do not set ``xavier_cache_bytes``. Relaunch after a worker restart. Native NIXL comparisons use the same engine on both roles. MLX is unsupported. Native NIXL backend ------------------- diff --git a/xinference/core/replica_config.py b/xinference/core/replica_config.py index a8fe95327e..3e2596ebfe 100644 --- a/xinference/core/replica_config.py +++ b/xinference/core/replica_config.py @@ -192,6 +192,9 @@ def validate_pd_replica_configs( "model_type", "model_path", "n_gpu", + "n_worker", + "shard", + "driver_info", "gpu_idx", "worker_ip", "replica", diff --git a/xinference/core/supervisor.py b/xinference/core/supervisor.py index b0a152f055..10b9de6362 100644 --- a/xinference/core/supervisor.py +++ b/xinference/core/supervisor.py @@ -3184,19 +3184,22 @@ async def launch_builtin_model( raise ValueError( "xavier_cache_bytes requires MLX Xavier or SGLang shared CPU caching and a positive integer" ) - if sglang_xavier: - if n_worker != 1: - raise ValueError( - "SGLang Xavier requires the xavier transport and one worker per replica" - ) + if sglang_xavier and n_worker != 1: + raise ValueError( + "SGLang Xavier requires the xavier transport and one worker per replica" + ) + if sglang_xavier or heterogeneous_pd: if ( model_type not in (None, "LLM") or model_format not in (None, "pytorch") or quantization not in (None, "none") ): raise ValueError( - "SGLang Xavier requires unquantized PyTorch LLM weights" + "Cross-engine Xavier PD requires unquantized PyTorch LLM weights" + if heterogeneous_pd + else "SGLang Xavier requires unquantized PyTorch LLM weights" ) + if sglang_xavier: kwargs["_xavier_cache_config"] = { "address": self.address, "uid": f"xavier-cache-{model_uid}", diff --git a/xinference/core/tests/test_pd_launch.py b/xinference/core/tests/test_pd_launch.py index 4eeac1bdf4..b077129886 100644 --- a/xinference/core/tests/test_pd_launch.py +++ b/xinference/core/tests/test_pd_launch.py @@ -796,6 +796,9 @@ async def test_cross_engine_mlx_is_rejected_before_allocating_actors( "model_engine", "xavier_config", "n_gpu", + "n_worker", + "shard", + "driver_info", "gpu_idx", "envs", "_xavier_cache_config", @@ -842,3 +845,32 @@ async def test_cross_engine_rejects_native_transport_before_launch(launch_runtim assert not actors for worker in workers: worker.launch_builtin_model.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "options,error", + [ + ({"model_format": "ggufv2"}, "unquantized PyTorch"), + ({"model_format": "mlx"}, "unquantized PyTorch"), + ({"quantization": "4bit"}, "unquantized PyTorch"), + ({"n_worker": 2}, "n_worker"), + ({"xavier_gpu_cache_bytes": 1024}, "retained GPU history"), + ({"xavier_cache_bytes": 1024}, "xavier_cache_bytes requires"), + ], +) +async def test_cross_engine_invalid_launch_has_no_side_effects( + launch_runtime, options, error +): + supervisor, workers, actors, _ = launch_runtime + kwargs = launch_kwargs() + kwargs["replica_config"][1].model_engine = "SGLang" + kwargs.update(options) + with pytest.raises(ValueError, match=error): + await supervisor.launch_builtin_model(**kwargs) + assert not actors and not supervisor._model_uid_to_replica_info + assert not supervisor._xavier_cache_mapping and not supervisor._pd_model_mapping + supervisor._status_guard_ref.update_replica_status.assert_not_awaited() + for worker in workers: + worker.launch_builtin_model.assert_not_awaited() + worker.wait_for_load.assert_not_awaited() diff --git a/xinference/core/tests/test_pd_model.py b/xinference/core/tests/test_pd_model.py index f4a6701df1..1f0181c9fb 100644 --- a/xinference/core/tests/test_pd_model.py +++ b/xinference/core/tests/test_pd_model.py @@ -231,6 +231,8 @@ async def pending(*args, **kwargs): {"prompt_logprobs": 0}, {"guided_choice": ["yes", "no"]}, {"json_schema": {"type": "object"}}, + {"response_format": {"type": "json_object"}}, + {"response_format": {"type": "json_schema", "json_schema": {"schema": {}}}}, {"return_logprob": True}, ], ) diff --git a/xinference/model/llm/vllm/tests/test_pd.py b/xinference/model/llm/vllm/tests/test_pd.py index 17ff2b4090..96a33094ab 100644 --- a/xinference/model/llm/vllm/tests/test_pd.py +++ b/xinference/model/llm/vllm/tests/test_pd.py @@ -12,6 +12,20 @@ from ..xavier.transport import normalize_xavier_transport_backend +@pytest.mark.parametrize("heterogeneous", [False, True]) +def test_set_xavier_config_preserves_only_heterogeneous_launch_config(heterogeneous): + from ..core import VLLMModel + + model = VLLMModel.__new__(VLLMModel) + configured = {"heterogeneous": heterogeneous, "role": "prefill"} + model._xavier_config = configured + model.set_xavier_config(None) + assert model._xavier_config == (configured if heterogeneous else None) + replacement = {"rank": 2} + model.set_xavier_config(replacement) + assert model._xavier_config is replacement + + def test_nixl_engine_config(monkeypatch): monkeypatch.setitem( sys.modules, "vllm.config", SimpleNamespace(KVTransferConfig=SimpleNamespace) diff --git a/xinference/model/llm/vllm/xavier/cross_engine.py b/xinference/model/llm/vllm/xavier/cross_engine.py index 4782430864..8c264c35e2 100644 --- a/xinference/model/llm/vllm/xavier/cross_engine.py +++ b/xinference/model/llm/vllm/xavier/cross_engine.py @@ -68,6 +68,7 @@ class CrossEngineMetadata(KVConnectorMetadata): direct_store: bool = False loads: list[tuple[str, int, list[int]]] = field(default_factory=list) errors: dict[str, str] = field(default_factory=dict) + pending_aborts: dict[str, int] = field(default_factory=dict) class CrossEngineConnector(XavierConnector): @@ -84,7 +85,6 @@ def __init__(self, vllm_config, role, kv_cache_config): self._directory = None self._cross_requests = {} self._cross_allocated = {} - self._prepared = set() self._cross_prepare_errors = {} self._pending_aborts = {} @@ -203,9 +203,12 @@ def update_state_after_alloc(self, request, blocks, num_external_tokens): def build_connector_meta(self, scheduler_output): meta = CrossEngineMetadata( - self._direct_sends, bool(scheduler_output.num_scheduled_tokens) + self._direct_sends, + bool(scheduler_output.num_scheduled_tokens), + pending_aborts=self._pending_aborts, ) self._direct_sends = set() + self._pending_aborts = {} for request_id, (room, destinations) in self._cross_allocated.items(): meta.loads.append((request_id, room, destinations)) if request_id in self._cross_prepare_errors: @@ -216,6 +219,7 @@ def build_connector_meta(self, scheduler_output): def start_load_kv(self, forward_context, **kwargs): meta = self._get_connector_metadata() self._direct_sends.update(meta.direct_sends) + self._pending_aborts.update(meta.pending_aborts) for request_id, room, destinations in meta.loads: async def submit( @@ -226,8 +230,11 @@ async def receive(): if request_id in meta.errors: raise ValueError(meta.errors[request_id]) actor = await self._ensure_gpu_cache_mapping() - return await self._receive_pages(actor, room, destinations) + return await actor.receive(room, destinations, 0) except Exception: + # The actor fences writes before returning an error. + # Report it per request; raising here kills EngineCore. + # kv_load_failure_policy=fail prevents local prefill. logger.warning( "Cross-engine KV load submission failed", exc_info=True ) @@ -238,19 +245,6 @@ async def receive(): self._call(submit()) - async def _receive_pages(self, actor, room, destinations): - try: - return await actor.receive(room, destinations, 0) - except Exception: - # The transfer actor fences writes before reporting an error. Return - # load errors through vLLM's per-request callback, including after an - # abort removes the room. Raising from get_finished kills EngineCore. - # kv_load_failure_policy=fail prevents fallback to local prefill. - logger.warning( - "Cross-engine KV load failed for room %s", room, exc_info=True - ) - return set(destinations) - def save_kv_layer(self, layer_name, kv_layer, attn_metadata, **kwargs): pass @@ -302,12 +296,10 @@ def request_finished(self, request, block_ids): self._pending_aborts[request.request_id] = room return hold, {"xavier_error": str(error)} self._direct_sends.add(request.request_id) - self._prepared.discard(request.request_id) return True, dict( do_remote_prefill=True, sglang_xavier=request.kv_transfer_params["sglang_xavier"], ) - self._prepared.discard(request.request_id) return False, None def get_finished(self, finished_req_ids): @@ -324,6 +316,8 @@ def get_finished(self, finished_req_ids): self._pending_aborts.pop(request_id) try: sent, received = super().get_finished(finished_req_ids) + for request_id in sent: + self._pending_aborts.pop(request_id, None) return drained | sent, received except Exception: # Preserve pinned requests for a retry if the actor RPC is unavailable. diff --git a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py index 92dac29124..c56fd582b8 100644 --- a/xinference/model/llm/vllm/xavier/test/test_cross_engine.py +++ b/xinference/model/llm/vllm/xavier/test/test_cross_engine.py @@ -3,6 +3,7 @@ import asyncio import importlib.util import json +import pickle import sys from pathlib import Path from types import SimpleNamespace @@ -36,7 +37,6 @@ def decode(cross_module, tokens): instance._cross_allocated = {} instance._direct_sends = set() instance._direct_handoff = True - instance._prepared = set() instance._cross_prepare_errors = {} instance._pending_aborts = {} instance._call = lambda coro: coro @@ -49,13 +49,15 @@ def decode(cross_module, tokens): return instance, request -def test_vllm_decode_reuses_all_but_final_prompt_token(cross_module): +@pytest.mark.parametrize("allocated_pages", [3, 4]) +def test_vllm_decode_reuses_all_but_final_prompt_token(cross_module, allocated_pages): instance, request = decode(cross_module, list(range(193))) assert instance.get_num_new_matched_tokens(request, 0) == (192, True) - blocks = SimpleNamespace(get_block_ids=lambda: ([9, 5, 7],)) + destinations = [9, 5, 7, 8][:allocated_pages] + blocks = SimpleNamespace(get_block_ids=lambda: (destinations,)) instance.update_state_after_alloc(request, blocks, 192) meta = instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) - assert meta.loads == [("r", 123, [9, 5, 7])] + assert meta.loads == [("r", 123, destinations)] assert request.kv_transfer_params["do_remote_prefill"] is False assert instance.get_num_new_matched_tokens(request, 0) == (0, False) # Preemption subsequently recomputes locally; the consumed room is single-use. @@ -73,7 +75,6 @@ async def test_cross_prepare_checks_actual_token_ids_once(cross_module): instance = cross_module.CrossEngineConnector.__new__( cross_module.CrossEngineConnector ) - instance._prepared = set() instance.contract = SimpleNamespace(fingerprint="a" * 64) instance._xavier_config = dict(role="decode", rank=1) directory = AsyncMock() @@ -122,20 +123,31 @@ async def test_failed_load_reports_all_blocks_without_crashing_vllm( instance, _ = decode(cross_module, list(range(193))) instance._loop = object() instance._invalid_block_ids = set() - instance._call = lambda coro: coro.close() actor = AsyncMock() actor.receive.side_effect = error - task = asyncio.create_task(instance._receive_pages(actor, 123, [9, 5, 7, 8])) - instance._gpu_load_jobs = {task: ["r"]} - await task + instance._ensure_gpu_cache_mapping = AsyncMock(return_value=actor) + instance._gpu_load_jobs = {} + + async def load(request_id, room, destinations): + meta = cross_module.CrossEngineMetadata( + loads=[(request_id, room, destinations)] + ) + instance._get_connector_metadata = lambda: meta + coroutines = [] + instance._call = coroutines.append + instance.start_load_kv(None) + for coroutine in coroutines: + await coroutine + await asyncio.gather(*instance._gpu_load_jobs) + instance._call = lambda coro: coro.close() + + await load("r", 123, [9, 5, 7, 8]) assert instance.get_finished(set()) == (set(), {"r"}) assert instance.get_block_ids_with_load_errors() == {9, 5, 7, 8} assert not instance._gpu_load_jobs actor.receive.side_effect = None actor.receive.return_value = set() - task = asyncio.create_task(instance._receive_pages(actor, 124, [1, 2])) - instance._gpu_load_jobs = {task: ["next"]} - await task + await load("next", 124, [1, 2]) assert instance.get_finished(set()) == (set(), {"next"}) assert instance.get_block_ids_with_load_errors() == set() @@ -198,11 +210,49 @@ def test_producer_registration_failure_is_drained_without_engine_exception( hold, metadata = instance.request_finished(request, [9]) assert hold and metadata["xavier_error"] == "reply lost" assert instance._pending_aborts == {"r": 123} - instance._direct_handoff = False - instance._gpu_load_jobs = {} - actor.abort.side_effect = None - assert instance.get_finished(set()) == ({"r"}, set()) + worker, _ = decode(cross_module, [1, 2]) + worker._is_producer, worker._is_consumer = True, False + worker._gpu_load_jobs = {} + worker._transfer_ref = actor + worker._get_transfer_ref = AsyncMock(return_value=actor) + worker._call = asyncio.run + actor.poll_direct_gpu_v1.return_value = set() + meta = pickle.loads( + pickle.dumps( + instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) + ) + ) assert not instance._pending_aborts and not instance._direct_sends + assert not worker._pending_aborts + worker._get_connector_metadata = lambda: meta + worker.start_load_kv(None) + assert worker._pending_aborts == {"r": 123} + assert worker.get_finished(set()) == (set(), set()) + assert worker._pending_aborts == {"r": 123} and worker._direct_sends == {"r"} + meta = instance.build_connector_meta(SimpleNamespace(num_scheduled_tokens={})) + worker.start_load_kv(None) + assert worker._pending_aborts == {"r": 123} + actor.abort.side_effect = None + assert worker.get_finished(set()) == ({"r"}, set()) + assert not worker._pending_aborts and not worker._direct_sends + assert worker.get_finished(set()) == (set(), set()) + + +def test_completed_send_clears_pending_abort_without_duplicate_finish(cross_module): + worker, _ = decode(cross_module, [1, 2]) + worker._is_producer, worker._is_consumer = True, False + worker._direct_sends = {"r"} + worker._pending_aborts = {"r": 123} + worker._gpu_load_jobs = {} + actor = AsyncMock() + actor.abort.side_effect = OSError("RPC unavailable") + actor.poll_direct_gpu_v1.return_value = {"r"} + worker._transfer_ref = actor + worker._get_transfer_ref = AsyncMock(return_value=actor) + worker._call = asyncio.run + assert worker.get_finished(set()) == ({"r"}, set()) + assert not worker._pending_aborts and not worker._direct_sends + assert worker.get_finished(set()) == (set(), set()) def test_completion_rpc_failure_retains_pinned_request_for_retry(cross_module): diff --git a/xinference/model/llm/vllm/xavier/test/test_v1_engine.py b/xinference/model/llm/vllm/xavier/test/test_v1_engine.py index c82a1f8a01..94fcaafbee 100644 --- a/xinference/model/llm/vllm/xavier/test/test_v1_engine.py +++ b/xinference/model/llm/vllm/xavier/test/test_v1_engine.py @@ -82,6 +82,25 @@ def test_v0_uses_legacy_adapter(engine, monkeypatch): factory.assert_not_called() +@pytest.mark.parametrize("role", ["prefill", "decode"]) +def test_heterogeneous_engine_selects_cross_connector_and_fail_policy(engine, role): + module, factory = engine + args = SimpleNamespace(create_model_config=lambda: SimpleNamespace(is_hybrid=False)) + module.XavierEngine.from_engine_args( + args, + xavier_config={"role": role, "heterogeneous": True, "gpu_cache_bytes": 0}, + ) + transfer = args.kv_transfer_config + assert transfer.kv_connector == "CrossEngineConnector" + assert ( + transfer.kv_connector_module_path + == "xinference.model.llm.vllm.xavier.cross_engine" + ) + assert transfer.kv_load_failure_policy == "fail" + assert transfer.kv_role == ("kv_producer" if role == "prefill" else "kv_consumer") + factory.assert_called_once() + + def test_incompatible_v1_fails_before_engine_start(engine): module, factory = engine module.VLLM_VERSION = "0.11.0" diff --git a/xinference/model/llm/xavier/backends/torch/pd.py b/xinference/model/llm/xavier/backends/torch/pd.py index 9e336303bb..a10d3291ac 100644 --- a/xinference/model/llm/xavier/backends/torch/pd.py +++ b/xinference/model/llm/xavier/backends/torch/pd.py @@ -200,16 +200,18 @@ async def poll_direct_gpu_v1(self): ) return finished - def init(self, room, count, aux_index): - state = self.rooms[room] - if state.get("aborted"): + def _producer_state(self, room): + state = self.rooms.get(room) + if state is None: raise RuntimeError("Cross-engine producer was cancelled") + return state + + def init(self, room, count, aux_index): + state = self._producer_state(room) state.update(total=count, aux_index=aux_index) async def add_chunk(self, room, pages, aux_payload: list[bytes] | None = None): - state = self.rooms[room] - if state.get("aborted"): - raise RuntimeError("Cross-engine producer was cancelled") + state = self._producer_state(room) state["sent"] += len(pages) state["deadline"] = time.monotonic() + transfer_timeout() final = state["sent"] == state["total"] @@ -235,13 +237,11 @@ async def add_chunk(self, room, pages, aux_payload: list[bytes] | None = None): state["changed"].set() def chunk(self, room, index): - state = self.rooms.get(room) - if state is None or state.get("aborted"): - raise RuntimeError("SGLang Xavier producer was cancelled") + state = self._producer_state(room) return state["chunks"][index] if index < len(state["chunks"]) else None async def wait_chunk(self, room, index): - state = self.rooms[room] + state = self._producer_state(room) state["changed"].clear() chunk = self.chunk(room, index) while chunk is None: @@ -290,13 +290,13 @@ async def release_remote_direct_gpu_v1(self, rank, ticket): def done(self, room): self.transfer.poll_direct() - state = self.rooms[room] + state = self._producer_state(room) return state["sent"] == state["total"] and len(state["released"]) == len( state["chunks"] ) async def wait_done(self, room): - state = self.rooms[room] + state = self._producer_state(room) while not state["completed"].is_set(): deadline = state["deadline"] try: diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py index 01a1d5546b..52744c6fdc 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine.py @@ -157,14 +157,27 @@ async def test_late_source_publication_does_not_leak_a_room(): @pytest.mark.asyncio -async def test_aborted_source_cannot_register_late_gpu_chunks(): - actor = CrossEngineGPUActor(None, None, None, "namespace", 1) - actor.rooms[123] = {"aborted": True} +async def test_aborted_source_cannot_register_late_gpu_chunks(kv_contract): + actor = cpu_actor(kv_contract, AsyncMock(), 1) + await actor.open(123) + actor.init(123, 3, 0) + await actor.add_chunk(123, [1]) + await actor.abort(123) + assert not actor.rooms with pytest.raises(RuntimeError, match="cancelled"): actor.init(123, 3, 0) with pytest.raises(RuntimeError, match="cancelled"): await actor.add_chunk(123, [1, 2, 3]) - assert actor.rooms[123] == {"aborted": True} + with pytest.raises(RuntimeError, match="cancelled"): + actor.done(123) + with pytest.raises(RuntimeError, match="cancelled"): + actor.chunk(123, 0) + with pytest.raises(RuntimeError, match="cancelled"): + await actor.wait_chunk(123, 0) + with pytest.raises(RuntimeError, match="cancelled"): + await actor.wait_done(123) + assert not actor.rooms + actor.transfer.register_direct.assert_called_once() @pytest.mark.asyncio @@ -188,8 +201,9 @@ async def test_bootstrap_abort_clears_room_and_wakes_existing_waiter(): @pytest.mark.asyncio @pytest.mark.parametrize("prompt_tokens", [64, 65, 128, 129, 193]) @pytest.mark.parametrize("split_final", [False, True]) +@pytest.mark.parametrize("full_allocation", [False, True]) async def test_vllm_imports_only_allocated_prefix_pages( - kv_contract, monkeypatch, prompt_tokens, split_final + kv_contract, monkeypatch, prompt_tokens, split_final, full_allocation ): directory = AsyncMock() directory.request_info.return_value = dict(prompt_tokens=prompt_tokens) @@ -200,7 +214,8 @@ async def test_vllm_imports_only_allocated_prefix_pages( target_count = (prompt_tokens - 1 + 63) // 64 source.caches = {"K": torch.arange(7 * 64, dtype=torch.float16).reshape(7, 64)} destination.caches = {"K": torch.zeros(7, 64, dtype=torch.float16)} - pages, targets = [5, 1, 3, 0][:source_count], [2, 6, 4, 0][:target_count] + allocated_count = source_count if full_allocation else target_count + pages, targets = [5, 1, 3, 0][:source_count], [2, 6, 4, 0][:allocated_count] await source.open(123) source.init(123, len(pages), 0) if split_final and len(pages) > 1: @@ -227,8 +242,10 @@ async def run(fn, *args): destination.transfer.load_direct, destination.transfer.run = load, run assert await destination.receive(123, targets, 0) == set() assert torch.equal( - destination.caches["K"][targets], source.caches["K"][pages[:target_count]] + destination.caches["K"][targets[:target_count]], + source.caches["K"][pages[:target_count]], ) + assert (destination.caches["K"][targets[target_count:]] == 0).all() assert source.rooms[123]["completed"].is_set() assert source.transfer.release_direct.call_count == len(source.rooms[123]["chunks"]) directory.complete.assert_awaited_once_with( diff --git a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py index 1cfeb72f5e..857494b4e8 100644 --- a/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py +++ b/xinference/model/llm/xavier/tests/test_pd_cross_engine_gpu.py @@ -23,10 +23,12 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("producer", ["vllm", "sglang"]) +@pytest.mark.parametrize( + "producer,full_allocation", [("vllm", False), ("sglang", False), ("sglang", True)] +) @pytest.mark.parametrize("prompt_tokens", [65, 129, 193]) async def test_gpu_pages_survive_cross_engine_layouts_bitwise( - kv_contract, producer, prompt_tokens + kv_contract, producer, full_allocation, prompt_tokens ): from torch.multiprocessing.reductions import reduce_tensor from xoscar.backends.allocate_strategy import ProcessIndex @@ -98,10 +100,11 @@ async def test_gpu_pages_survive_cross_engine_layouts_bitwise( target_count = ( source_count if producer == "vllm" else (prompt_tokens - 1 + 63) // 64 ) + allocated_count = source_count if full_allocation else target_count room, source, destination = ( 123, [5, 1, 3, 0][:source_count], - [2, 6, 4, 0][:target_count], + [2, 6, 4, 0][:allocated_count], ) for rank, role in enumerate(("prefill", "decode"), 1): await directory.prepare( @@ -134,8 +137,10 @@ async def test_gpu_pages_survive_cross_engine_layouts_bitwise( } for key, original in views[0].items(): assert torch.equal( - original[source[:target_count]].cpu(), views[1][key][destination].cpu() + original[source[:target_count]].cpu(), + views[1][key][destination[:target_count]].cpu(), ) + assert views[1][key][destination[target_count:]].count_nonzero().item() == 0 stats = await directory.get_stats() assert ( stats["imported_tokens"]